diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..381459476 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +/start-server.sh text eol=lf +/start-server.bat text eol=crlf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9539e9a33..35418c280 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,6 +26,8 @@ on: "ggml", "examples/server/frontend", "examples/server/frontend/**", + "start-server.sh", + "start-server.bat", ] pull_request: types: [opened, synchronize, reopened] @@ -44,6 +46,8 @@ on: "ggml", "examples/server/frontend", "examples/server/frontend/**", + "start-server.sh", + "start-server.bat", ] env: @@ -107,7 +111,8 @@ jobs: run: | cp ggml/LICENSE ./build/bin/ggml.txt cp LICENSE ./build/bin/stable-diffusion.cpp.txt - zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip ./build/bin/* + chmod +x start-server.sh + zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip ./build/bin/* ./start-server.sh ./start-server.bat - name: Upload artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} @@ -171,7 +176,8 @@ jobs: run: | cp ggml/LICENSE ./build/bin/ggml.txt cp LICENSE ./build/bin/stable-diffusion.cpp.txt - zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-vulkan.zip ./build/bin/* + chmod +x start-server.sh + zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-vulkan.zip ./build/bin/* ./start-server.sh ./start-server.bat - name: Upload artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} @@ -326,7 +332,8 @@ jobs: run: | cp ggml/LICENSE ./build/bin/ggml.txt cp LICENSE ./build/bin/stable-diffusion.cpp.txt - zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip ./build/bin/* + chmod +x start-server.sh + zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip ./build/bin/* ./start-server.sh ./start-server.bat - name: Upload artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} @@ -418,7 +425,7 @@ jobs: ls .\build\bin throw "Can't find stable-diffusion.dll" } - 7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip $filePath + 7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip $filePath .\start-server.sh .\start-server.bat - name: Copy and pack Cuda runtime id: pack_cuda_runtime @@ -579,7 +586,7 @@ jobs: - name: Pack artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} run: | - 7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip .\build\bin\* + 7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip .\build\bin\* .\start-server.sh .\start-server.bat - name: Upload artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} @@ -743,7 +750,8 @@ jobs: run: | cp ggml/LICENSE ./build/bin/ggml.txt cp LICENSE ./build/bin/stable-diffusion.cpp.txt - zip -y -r sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm-${{ matrix.ROCM_VERSION }}.zip ./build/bin + chmod +x start-server.sh + zip -y -r sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm-${{ matrix.ROCM_VERSION }}.zip ./build/bin ./start-server.sh ./start-server.bat - name: Upload artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} diff --git a/.gitignore b/.gitignore index 80599b3f8..b55ea0f81 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ tests/ .cache/ *.swp *.bat +!/start-server.bat +/start-server.local.sh +/start-server.local.bat +/start-server.local.ps1 *.bin *.exe *.gguf diff --git a/AGENTS.md b/AGENTS.md index 4953e2278..a48040869 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -83,6 +83,8 @@ documentation. and debugging. * `examples/common/` - shared example support code. * `examples/server/` - server application built on top of the library. + Before modifying built-in recipes, read and follow + [the built-in recipe policy](examples/server/recipes/README.md). * `examples/server/frontend/` - git submodule containing independent frontend code. Avoid modifying it unless explicitly requested. diff --git a/CMakeLists.txt b/CMakeLists.txt index bd7c6d9e9..d7bfda599 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -352,9 +352,9 @@ if (SD_HIPBLAS) set_property(SOURCE src/core/ggml_extend_backend.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_HIP_MFMA) endif() endif() -target_include_directories(${SD_LIB} PUBLIC . src include) -target_include_directories(${SD_LIB} PRIVATE src/core) -target_include_directories(${SD_LIB} PUBLIC . thirdparty) +target_include_directories(${SD_LIB} PUBLIC include) +target_include_directories(${SD_LIB} PRIVATE src src/core) +target_include_directories(${SD_LIB} PUBLIC thirdparty) target_compile_features(${SD_LIB} PUBLIC c_std_11 cxx_std_17) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1f4c06792..ca64ad198 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,6 +33,17 @@ PRs should include: The [UIs list in the README](README.md#uis) only includes open-source projects that use `stable-diffusion.cpp` as a backend. Do not submit PRs to add closed-source UIs. +## Server Recipes + +Before adding or changing a built-in server recipe, read and follow the +[built-in recipe policy](examples/server/recipes/README.md). The policy governs +native editing, community editing LoRAs, distinct task checkpoints, external +adapters, native components and layer-count variants. Do not add recipes solely +for style or acceleration LoRAs, separately stored native components, or added +or removed model layers. Explain the task, component compatibility or capability +difference that justifies a new recipe, and verify its required components, +feature declarations and embedded API representation. + ## Code Style Format code according to the repository style before submitting changes. diff --git a/README.md b/README.md index abb2dcb20..b4ab2f5ab 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ API and command-line option may change frequently.*** ## πŸ”₯Important News +* **2026/10/16** πŸš€ stable-diffusion.cpp adds **Day-0 support for MOSS-Image-Base / MOSS-Image-SFT** * **2026/10/16** πŸš€ stable-diffusion.cpp adds **Day-0 support for FLUX.3 Image Klein / Klein Base** * **2026/09/20** πŸš€ stable-diffusion.cpp adds **Day-0 support for Qwen-Image-2.1** * **2026/08/20** πŸš€ stable-diffusion.cpp now supports **LTX-2.5** @@ -46,6 +47,7 @@ API and command-line option may change frequently.*** - [FLUX.1-dev/FLUX.1-schnell](./docs/flux.md) - [FLUX.2-dev/FLUX.2-klein](./docs/flux2.md) - [FLUX.3 Image Klein / Klein Base](./docs/flux3.md) + - [MOSS-Image-Base / MOSS-Image-SFT](./docs/moss_image.md) - [Lens](./docs/lens.md) - [Chroma](./docs/chroma.md) - [Chroma1-Radiance](./docs/chroma_radiance.md) diff --git a/assets/moss/sft.png b/assets/moss/sft.png new file mode 100644 index 000000000..ede42d1f7 Binary files /dev/null and b/assets/moss/sft.png differ diff --git a/docker/Dockerfile.musa b/docker/Dockerfile.musa index 7dda431aa..08d6f1d43 100644 --- a/docker/Dockerfile.musa +++ b/docker/Dockerfile.musa @@ -1,7 +1,6 @@ -ARG MUSA_VERSION=rc4.2.0 -ARG UBUNTU_VERSION=22.04 +ARG MUSA_IMAGE=registry.mthreads.com/mcconline/inference/pytorch:2.9.1.post1-py3.10-musa5.2.0-mp31-devel-ubuntu22.04-amd64 -FROM mthreads/musa:${MUSA_VERSION}-devel-ubuntu${UBUNTU_VERSION}-amd64 as build +FROM ${MUSA_IMAGE} AS build # sd-server embeds the web UI at build time, so the build image needs Node/pnpm. RUN apt-get update && apt-get install -y --no-install-recommends ccache cmake git ca-certificates curl gnupg && \ @@ -22,9 +21,8 @@ COPY . . RUN mkdir build && cd build && \ cmake .. -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS} -fopenmp -I/usr/lib/llvm-14/lib/clang/14.0.0/include -L/usr/lib/llvm-14/lib" \ - -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -fopenmp -I/usr/lib/llvm-14/lib/clang/14.0.0/include -L/usr/lib/llvm-14/lib" \ -DSD_MUSA=ON \ + -DMUSA_ARCHITECTURES=31 \ -DSD_BUILD_SHARED_LIBS=ON \ -DGGML_NATIVE=OFF \ -DSD_BUILD_SHARED_GGML_LIB=ON \ @@ -35,7 +33,7 @@ RUN mkdir build && cd build && \ -DCMAKE_BUILD_TYPE=Release && \ cmake --build . --config Release -FROM mthreads/musa:${MUSA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}-amd64 as runtime +FROM ${MUSA_IMAGE} AS runtime COPY --from=build /sd.cpp/build/bin /sd.cpp/bin RUN printf '#!/bin/sh\nexec /sd.cpp/bin/sd-cli "$@"\n' > /sd-cli && \ @@ -45,3 +43,4 @@ RUN printf '#!/bin/sh\nexec /sd.cpp/bin/sd-cli "$@"\n' > /sd-cli && \ WORKDIR /sd.cpp ENTRYPOINT [ "/sd-cli" ] +CMD [] diff --git a/docs/build.md b/docs/build.md index 9daaa219b..6ec0e40f0 100644 --- a/docs/build.md +++ b/docs/build.md @@ -119,14 +119,16 @@ cmake --build . --config Release ## Build with MUSA -This provides GPU acceleration using Moore Threads GPU. Make sure to have the MUSA toolkit installed. +This provides GPU acceleration using Moore Threads GPUs. The configuration below follows upstream validation with MUSA SDK 5.2.0 and MTT S5000 (PH1, `mp_31`). Install that toolkit before building. Older SDKs such as 4.2 may lack the CUB headers required by the current backend. ```shell mkdir build && cd build -cmake .. -DCMAKE_C_COMPILER=/usr/local/musa/bin/clang -DCMAKE_CXX_COMPILER=/usr/local/musa/bin/clang++ -DSD_MUSA=ON -DCMAKE_BUILD_TYPE=Release +cmake .. -DCMAKE_C_COMPILER=/usr/local/musa/bin/clang -DCMAKE_CXX_COMPILER=/usr/local/musa/bin/clang++ -DSD_MUSA=ON -DMUSA_ARCHITECTURES=31 -DCMAKE_BUILD_TYPE=Release cmake --build . --config Release ``` +The MUSA Docker image uses the same SDK 5.2.0 development image as upstream for both build and runtime stages, and targets `mp_31` only. It includes the development tools and PyTorch from that base image, so it is larger than a minimal runtime image. Other GPU architectures require separate validation. + ## Build with Metal Using Metal makes the computation run on the GPU. Currently, there are some issues with Metal when performing operations on very large matrices, making it highly inefficient at the moment. Performance improvements are expected in the near future. diff --git a/docs/flux3.md b/docs/flux3.md index dd11e4044..8bc0b2ec2 100644 --- a/docs/flux3.md +++ b/docs/flux3.md @@ -6,8 +6,10 @@ FLUX.3 Image Klein uses a FLUX.3 diffusion transformer, the FLUX.2 VAE, and Qwen - Download FLUX.3 Image Klein - safetensors: https://huggingface.co/black-forest-labs/flux-3-image-klein/tree/main + - gguf: https://huggingface.co/leejet/flux-3-image-klein-GGUF/tree/main - Download FLUX.3 Image Klein Base - safetensors: https://huggingface.co/black-forest-labs/flux-3-image-klein-base/tree/main + - gguf: https://huggingface.co/leejet/flux-3-image-klein-base-GGUF/tree/main - Download vae - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main - The `ae.safetensors` included with FLUX.3 can also be used. diff --git a/docs/moss_image.md b/docs/moss_image.md new file mode 100644 index 000000000..27ab26f80 --- /dev/null +++ b/docs/moss_image.md @@ -0,0 +1,37 @@ +# How to Use + +MOSS-Image uses a 5B diffusion transformer, the FLUX.2 VAE, and Qwen3.5 9B as the LLM text encoder. Both Base and SFT are supported. Prompt enhancement is not included. + +## Download weights + +- Download MOSS-Image Base or SFT + - safetensors: https://huggingface.co/Comfy-Org/MOSS-Image/tree/main/diffusion_models + - gguf: + - Base: https://huggingface.co/leejet/MOSS-Image-Base-GGUF/tree/main + - SFT: https://huggingface.co/leejet/MOSS-Image-SFT-GGUF/tree/main +- Download vae + - safetensors: https://huggingface.co/Comfy-Org/MOSS-Image/tree/main/vae +- Download Qwen3.5 9B + - gguf: https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/tree/main +- Download `tokenizer.json` + - https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base/tree/main/tokenizer + +Pass the external `tokenizer.json` with `--tokenizer`; Qwen3.5 uses a different vocabulary from the embedded Qwen tokenizer. + +## Examples + +### MOSS-Image Base + +``` +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\moss_image_base_int8_convrot_attn.safetensors --llm ..\models\text_encoders\Qwen3.5-9B-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors --tokenizer ..\models\tokenizers\tokenizer_qwen3.5.json --cfg-scale 3.0 -p "a lovely cat holding a sign says 'moss-image.cpp'" -v --offload-to-cpu -H 1024 -W 1024 +``` + +### MOSS-Image SFT + +``` +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\moss_image_sft_int8_convrot_attn.safetensors --llm ..\models\text_encoders\Qwen3.5-9B-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors --tokenizer ..\models\tokenizers\tokenizer_qwen3.5.json --cfg-scale 3.0 -p "a lovely cat holding a sign says 'moss-image.cpp'" -v --offload-to-cpu -H 1024 -W 1024 +``` + + + +Use dimensions divisible by 16. The default discrete scheduler uses the upstream time shift of 5. Use `--flow-shift` to override it. The negative CFG branch skips DiT layer 22 (zero-based), matching the upstream pipeline. diff --git a/examples/cli/image_metadata.cpp b/examples/cli/image_metadata.cpp index 8dd339c88..ba0d6da23 100644 --- a/examples/cli/image_metadata.cpp +++ b/examples/cli/image_metadata.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -276,7 +277,7 @@ namespace { } bool read_file(const std::string& path, std::vector& data, std::string& error) { - std::ifstream fin(path, std::ios::binary); + std::ifstream fin(std::filesystem::u8path(path), std::ios::binary); if (!fin) { error = "failed to open file: " + path; return false; diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index e8174d27c..ca8ce4e79 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -188,7 +188,7 @@ struct SDCliParams { if (++index >= argc) { return -1; } - imatrix_in.push_back(argv[index]); + imatrix_in.push_back(argv_to_utf8(index, argv)); return 1; }; @@ -323,13 +323,14 @@ void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) { bool load_images_from_dir(const std::string dir, std::vector& images, int max_image_num = 0) { - if (!fs::exists(dir) || !fs::is_directory(dir)) { + const fs::path dir_path = fs::u8path(dir); + if (!fs::exists(dir_path) || !fs::is_directory(dir_path)) { LOG_ERROR("'%s' is not a valid directory\n", dir.c_str()); return false; } std::vector entries; - for (const auto& entry : fs::directory_iterator(dir)) { + for (const auto& entry : fs::directory_iterator(dir_path)) { if (entry.is_regular_file()) { entries.push_back(entry); } @@ -337,12 +338,12 @@ bool load_images_from_dir(const std::string dir, std::sort(entries.begin(), entries.end(), [](const fs::directory_entry& a, const fs::directory_entry& b) { - return a.path().filename().string() < b.path().filename().string(); + return a.path().filename().u8string() < b.path().filename().u8string(); }); for (const auto& entry : entries) { - std::string path = entry.path().string(); - std::string ext = entry.path().extension().string(); + std::string path = entry.path().u8string(); + std::string ext = entry.path().extension().u8string(); std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower); if (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".bmp" || ext == ".webp") { @@ -397,19 +398,19 @@ void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy, // is_noisy is set to true if the preview corresponds to noisy latents, false if it's denoised latents // unused in this app, it will either be always noisy or always denoised here if (frame_count == 1) { - fs::path path = cli_params->preview_path; - if (encoded_image_format_from_path(path.string()) == EncodedImageFormat::UNKNOWN) + fs::path path = fs::u8path(cli_params->preview_path); + if (encoded_image_format_from_path(path.u8string()) == EncodedImageFormat::UNKNOWN) path += ".png"; - if (std::regex_search(path.string(), format_specifier_regex)) - path = fs::path(format_frame_idx(path.string(), continuous_preview_counter++)); - if (!write_image_to_file(path.string(), + if (std::regex_search(path.u8string(), format_specifier_regex)) + path = fs::u8path(format_frame_idx(path.u8string(), continuous_preview_counter++)); + if (!write_image_to_file(path.u8string(), image->data, image->width, image->height, image->channel, "", cli_params->compression_quality)) { - LOG_ERROR("save preview image to '%s' failed", path.string().c_str()); + LOG_ERROR("save preview image to '%s' failed", path.u8string().c_str()); } } else { int fps = cli_params->preview_method == PREVIEW_PROJ ? cli_params->preview_fps / 4 : cli_params->preview_fps; @@ -420,12 +421,12 @@ void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy, } static fs::path get_video_audio_sidecar_path(const SDCliParams& cli_params) { - fs::path out_path = cli_params.output_path; + fs::path out_path = fs::u8path(cli_params.output_path); fs::path base_path = out_path; fs::path ext = out_path.has_extension() ? out_path.extension() : fs::path{}; - std::string ext_lower = ext.string(); + std::string ext_lower = ext.u8string(); std::transform(ext_lower.begin(), ext_lower.end(), ext_lower.begin(), ::tolower); - const EncodedImageFormat output_format = encoded_image_format_from_path(out_path.string()); + const EncodedImageFormat output_format = encoded_image_format_from_path(out_path.u8string()); if (!ext.empty()) { if (output_format == EncodedImageFormat::JPEG || output_format == EncodedImageFormat::PNG || @@ -450,14 +451,14 @@ bool save_results(const SDCliParams& cli_params, } namespace fs = std::filesystem; - fs::path out_path = cli_params.output_path; + fs::path out_path = fs::u8path(cli_params.output_path); if (!out_path.parent_path().empty()) { std::error_code ec; fs::create_directories(out_path.parent_path(), ec); if (ec) { LOG_ERROR("failed to create directory '%s': %s", - out_path.parent_path().string().c_str(), ec.message().c_str()); + out_path.parent_path().u8string().c_str(), ec.message().c_str()); return false; } } @@ -465,9 +466,9 @@ bool save_results(const SDCliParams& cli_params, fs::path base_path = out_path; fs::path ext = out_path.has_extension() ? out_path.extension() : fs::path{}; - std::string ext_lower = ext.string(); + std::string ext_lower = ext.u8string(); std::transform(ext_lower.begin(), ext_lower.end(), ext_lower.begin(), ::tolower); - const EncodedImageFormat output_format = encoded_image_format_from_path(out_path.string()); + const EncodedImageFormat output_format = encoded_image_format_from_path(out_path.u8string()); if (!ext.empty()) { if (output_format == EncodedImageFormat::JPEG || output_format == EncodedImageFormat::PNG || @@ -493,8 +494,8 @@ bool save_results(const SDCliParams& cli_params, std::string params = gen_params.embed_image_metadata ? get_image_params(ctx_params, gen_params, metadata_seed, cli_params.mode) : ""; - const bool ok = write_image_to_file(path.string(), img.data, img.width, img.height, img.channel, params, cli_params.compression_quality); - LOG_INFO("save result image %d to '%s' (%s)", idx, path.string().c_str(), ok ? "success" : "failure"); + const bool ok = write_image_to_file(path.u8string(), img.data, img.width, img.height, img.channel, params, cli_params.compression_quality); + LOG_INFO("save result image %d to '%s' (%s)", idx, path.u8string().c_str(), ok ? "success" : "failure"); return ok; }; @@ -502,14 +503,14 @@ bool save_results(const SDCliParams& cli_params, if (generated_audio == nullptr) { return; } - if (write_wav_to_file(wav_path.string(), + if (write_wav_to_file(wav_path.u8string(), generated_audio->data, generated_audio->sample_count, generated_audio->channels, generated_audio->sample_rate)) { - LOG_INFO("save result audio to '%s'", wav_path.string().c_str()); + LOG_INFO("save result audio to '%s'", wav_path.u8string().c_str()); } else { - LOG_WARN("failed to save result audio to '%s'", wav_path.string().c_str()); + LOG_WARN("failed to save result audio to '%s'", wav_path.u8string().c_str()); } }; @@ -522,7 +523,7 @@ bool save_results(const SDCliParams& cli_params, pattern += ext; for (int i = 0; i < num_results; ++i) { - fs::path img_path = format_frame_idx(pattern.string(), output_begin_idx + i); + fs::path img_path = fs::u8path(format_frame_idx(pattern.u8string(), output_begin_idx + i)); if (write_image(img_path, i)) { sucessful_reults++; } @@ -540,11 +541,11 @@ bool save_results(const SDCliParams& cli_params, fs::path video_path = base_path; video_path += ext; - std::string final_ext_lower = ext.string(); + std::string final_ext_lower = ext.u8string(); std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower); const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm"); - if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) { - LOG_INFO("save result video to '%s'", video_path.string().c_str()); + if (create_video_from_sd_images(video_path.u8string().c_str(), results, num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) { + LOG_INFO("save result video to '%s'", video_path.u8string().c_str()); if (generated_audio != nullptr && !mux_audio) { fs::path wav_path = video_path; wav_path.replace_extension(".wav"); @@ -552,7 +553,7 @@ bool save_results(const SDCliParams& cli_params, } return true; } else { - LOG_ERROR("Failed to save result video to '%s'", video_path.string().c_str()); + LOG_ERROR("Failed to save result video to '%s'", video_path.u8string().c_str()); return false; } } @@ -701,7 +702,7 @@ int main(int argc, const char* argv[]) { LOG_DEBUG("%s", gen_params.to_string().c_str()); if (!cli_params.imatrix_out.empty()) { - if (fs::exists(cli_params.imatrix_out) && + if (fs::exists(fs::u8path(cli_params.imatrix_out)) && std::find(cli_params.imatrix_in.begin(), cli_params.imatrix_in.end(), cli_params.imatrix_out) == cli_params.imatrix_in.end()) { LOG_WARN("imatrix file '%s' already exists and will be overwritten", cli_params.imatrix_out.c_str()); } diff --git a/examples/common/common.cpp b/examples/common/common.cpp index 11b531f46..9e7851fef 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -75,7 +75,7 @@ static std::string utf16_to_utf8(const std::wstring& wstr) { return utf8; } -static std::string argv_to_utf8(int index, const char** argv) { +std::string argv_to_utf8(int index, const char** argv) { (void)argv; int argc; wchar_t** argv_w = CommandLineToArgvW(GetCommandLineW(), &argc); @@ -91,7 +91,7 @@ static std::string argv_to_utf8(int index, const char** argv) { } #else // Linux / macOS -static std::string argv_to_utf8(int index, const char** argv) { +std::string argv_to_utf8(int index, const char** argv) { return std::string(argv[index]); } @@ -839,16 +839,17 @@ ArgOptions SDContextParams::get_options() { void SDContextParams::build_embedding_map() { static const std::vector valid_ext = {".gguf", ".safetensors", ".pt", ".ckpt"}; - if (!fs::exists(embedding_dir) || !fs::is_directory(embedding_dir)) { + const fs::path embedding_dir_path = fs::u8path(embedding_dir); + if (!fs::exists(embedding_dir_path) || !fs::is_directory(embedding_dir_path)) { return; } - for (auto& p : fs::directory_iterator(embedding_dir)) { + for (auto& p : fs::directory_iterator(embedding_dir_path)) { if (!p.is_regular_file()) continue; auto path = p.path(); - std::string ext = path.extension().string(); + std::string ext = path.extension().u8string(); bool valid = false; for (auto& e : valid_ext) { @@ -860,8 +861,8 @@ void SDContextParams::build_embedding_map() { if (!valid) continue; - std::string key = path.stem().string(); - std::string value = path.string(); + std::string key = path.stem().u8string(); + std::string value = path.u8string(); embedding_map[key] = value; } @@ -1572,7 +1573,7 @@ ArgOptions SDGenerationParams::get_options() { if (++index >= argc) { return -1; } - ref_image_paths.push_back(argv[index]); + ref_image_paths.push_back(argv_to_utf8(index, argv)); return 1; }; @@ -1580,7 +1581,7 @@ ArgOptions SDGenerationParams::get_options() { if (++index >= argc) { return -1; } - ref_video_paths.push_back(argv[index]); + ref_video_paths.push_back(argv_to_utf8(index, argv)); return 1; }; @@ -1588,7 +1589,7 @@ ArgOptions SDGenerationParams::get_options() { if (++index >= argc) { return -1; } - ref_video_audio_paths.push_back(argv[index]); + ref_video_audio_paths.push_back(argv_to_utf8(index, argv)); return 1; }; @@ -1596,7 +1597,7 @@ ArgOptions SDGenerationParams::get_options() { if (++index >= argc) { return -1; } - ref_audio_paths.push_back(argv[index]); + ref_audio_paths.push_back(argv_to_utf8(index, argv)); return 1; }; @@ -1604,7 +1605,7 @@ ArgOptions SDGenerationParams::get_options() { if (++index >= argc) { return -1; } - ref_audio_paths.push_back(argv[index]); + ref_audio_paths.push_back(argv_to_utf8(index, argv)); return 1; }; @@ -1703,15 +1704,15 @@ ArgOptions SDGenerationParams::get_options() { if (++index >= argc) { return -1; } - const char* arg = argv[index]; - std::ifstream f(arg, std::ios::binary); + const std::string arg = argv_to_utf8(index, argv); + std::ifstream f(fs::u8path(arg), std::ios::binary); try { prompt = std::string(std::istreambuf_iterator{f}, {}); } catch (const std::ios_base::failure&) { f.setstate(std::ios_base::failbit); } if (f.fail()) { - LOG_ERROR("error: failed to read prompt file '%s'\n", arg); + LOG_ERROR("error: failed to read prompt file '%s'\n", arg.c_str()); return -1; } return 1; @@ -1721,15 +1722,15 @@ ArgOptions SDGenerationParams::get_options() { if (++index >= argc) { return -1; } - const char* arg = argv[index]; - std::ifstream f(arg, std::ios::binary); + const std::string arg = argv_to_utf8(index, argv); + std::ifstream f(fs::u8path(arg), std::ios::binary); try { negative_prompt = std::string(std::istreambuf_iterator{f}, {}); } catch (const std::ios_base::failure&) { f.setstate(std::ios_base::failbit); } if (f.fail()) { - LOG_ERROR("error: failed to read negative prompt file '%s'\n", arg); + LOG_ERROR("error: failed to read negative prompt file '%s'\n", arg.c_str()); return -1; } return 1; @@ -2050,22 +2051,22 @@ static bool resolve_model_file_from_dir(const std::string& model_name, if (model_name.empty() || model_name.find('/') != std::string::npos || model_name.find('\\') != std::string::npos || - fs::path(model_name).has_root_path() || + fs::u8path(model_name).has_root_path() || ends_with_valid_ext()) { LOG_ERROR("%s must be a model name without path or extension: %s", label, model_name.c_str()); return false; } - fs::path model_dir_path = model_dir; + fs::path model_dir_path = fs::u8path(model_dir); for (const auto& ext : valid_ext) { - fs::path try_path = model_dir_path / (model_name + ext); + fs::path try_path = model_dir_path / fs::u8path(model_name + ext); if (fs::exists(try_path) && fs::is_regular_file(try_path)) { - resolved_path = try_path.lexically_normal().string(); + resolved_path = try_path.lexically_normal().u8string(); return true; } } - LOG_ERROR("can not find %s %s in %s", label, model_name.c_str(), model_dir_path.lexically_normal().string().c_str()); + LOG_ERROR("can not find %s %s in %s", label, model_name.c_str(), model_dir_path.lexically_normal().u8string().c_str()); return false; } @@ -2217,7 +2218,8 @@ bool SDGenerationParams::from_json_str( } if (target_custom_sigmas != nullptr && sample_json.contains("custom_sigmas") && - sample_json["custom_sigmas"].is_array()) { + sample_json["custom_sigmas"].is_array() && + !sample_json["custom_sigmas"].empty()) { // empty [] = "not specified", keep server/CLI default *target_custom_sigmas = sample_json["custom_sigmas"].get>(); } if (sample_json.contains("sample_method") && sample_json["sample_method"].is_string()) { @@ -2359,10 +2361,18 @@ bool SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_d } fs::path final_path; - if (is_absolute_path(raw_path)) { - final_path = raw_path; - } else { - final_path = fs::path(lora_model_dir) / raw_path; + try { + if (is_absolute_path(raw_path)) { + final_path = fs::u8path(raw_path); + } else { + final_path = fs::u8path(lora_model_dir) / fs::u8path(raw_path); + } + } catch (const std::exception&) { // not UTF-8, e.g. a prompt file in another encoding + LOG_WARN("invalid lora path %s", raw_path.c_str()); + tmp = m.suffix().str(); + prompt = std::regex_replace(prompt, re, "", std::regex_constants::format_first_only); + missing = true; + continue; } if (!fs::exists(final_path)) { bool found = false; @@ -2376,7 +2386,7 @@ bool SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_d } } if (!found) { - LOG_WARN("can not found lora %s", final_path.lexically_normal().string().c_str()); + LOG_WARN("can not found lora %s", final_path.lexically_normal().u8string().c_str()); tmp = m.suffix().str(); prompt = std::regex_replace(prompt, re, "", std::regex_constants::format_first_only); missing = true; @@ -2384,7 +2394,7 @@ bool SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_d } } - const std::string key = final_path.lexically_normal().string(); + const std::string key = final_path.lexically_normal().u8string(); if (is_high_noise) high_noise_lora_map[key] += mul; diff --git a/examples/common/common.h b/examples/common/common.h index 74a057d0b..91e75c685 100644 --- a/examples/common/common.h +++ b/examples/common/common.h @@ -12,6 +12,10 @@ #include "resource_owners.hpp" #include "stable-diffusion.h" +// The command-line argument at `index` as UTF-8 (on Windows, read again from +// the UTF-16 command line: argv itself is in the ANSI code page). +std::string argv_to_utf8(int index, const char** argv); + #define SAFE_STR(s) ((s) ? (s) : "") #define BOOL_STR(b) ((b) ? "true" : "false") diff --git a/examples/common/media_io.cpp b/examples/common/media_io.cpp index ba0e8c5b2..baac6d200 100644 --- a/examples/common/media_io.cpp +++ b/examples/common/media_io.cpp @@ -15,10 +15,12 @@ #define STB_IMAGE_IMPLEMENTATION #define STB_IMAGE_STATIC +#define STBI_WINDOWS_UTF8 #include "stb_image.h" #define STB_IMAGE_WRITE_IMPLEMENTATION #define STB_IMAGE_WRITE_STATIC +#define STBIW_WINDOWS_UTF8 #include "stb_image_write.h" #define STB_IMAGE_RESIZE_IMPLEMENTATION @@ -147,7 +149,7 @@ class MemoryMkvWriter : public mkvmuxer::IMkvWriter { #endif bool read_binary_file_bytes(const char* path, std::vector& data) { - std::ifstream fin(fs::path(path), std::ios::binary); + std::ifstream fin(fs::u8path(path), std::ios::binary); if (!fin) { return false; } @@ -170,7 +172,7 @@ bool read_binary_file_bytes(const char* path, std::vector& data) { } bool write_binary_file_bytes(const std::string& path, const std::vector& data) { - std::ofstream fout(fs::path(path), std::ios::binary); + std::ofstream fout(fs::u8path(path), std::ios::binary); if (!fout) { return false; } @@ -714,7 +716,7 @@ static std::pair audio_sample_range_for_video_frame(const sd } EncodedImageFormat encoded_image_format_from_path(const std::string& path) { - std::string ext = fs::path(path).extension().string(); + std::string ext = fs::u8path(path).extension().u8string(); std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower); if (ext == ".jpg" || ext == ".jpeg" || ext == ".jpe") { @@ -1413,7 +1415,7 @@ bool write_wav_to_file(const std::string& path, return false; } - std::ofstream file(path, std::ios::binary); + std::ofstream file(fs::u8path(path), std::ios::binary); if (!file.is_open()) { return false; } @@ -1475,7 +1477,7 @@ bool load_wav_from_file(const std::string& path, sample_rate = 0; channels = 0; - std::ifstream file(path, std::ios::binary); + std::ifstream file(fs::u8path(path), std::ios::binary); uint8_t riff_header[12]; if (!file.read(reinterpret_cast(riff_header), sizeof(riff_header)) || std::memcmp(riff_header, "RIFF", 4) != 0 || diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt index 302a8d609..58260be0d 100644 --- a/examples/server/CMakeLists.txt +++ b/examples/server/CMakeLists.txt @@ -68,6 +68,8 @@ add_executable(${TARGET} ../common/media_io.cpp main.cpp runtime.cpp + model_catalog.cpp + routes_models.cpp async_jobs.cpp routes_index.cpp routes_openai.cpp @@ -88,6 +90,17 @@ else() message(STATUS "HAVE_INDEX_HTML disabled") endif() +file(GLOB BUILTIN_RECIPE_FILES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/recipes/*.json") +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${BUILTIN_RECIPE_FILES}) +# Keep recipes in separate literals to stay within MSVC's string size limit. +set(SD_BUILTIN_RECIPES "") +foreach(RECIPE_FILE IN LISTS BUILTIN_RECIPE_FILES) + file(READ "${RECIPE_FILE}" RECIPE_JSON) + string(APPEND SD_BUILTIN_RECIPES "R\"recipes(${RECIPE_JSON})recipes\",\n") +endforeach() +configure_file(recipes_builtin.h.in "${CMAKE_CURRENT_BINARY_DIR}/recipes_builtin.h" @ONLY) +target_include_directories(${TARGET} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") + install(TARGETS ${TARGET} RUNTIME) target_link_libraries(${TARGET} PRIVATE stable-diffusion ${CMAKE_THREAD_LIBS_INIT}) if(SD_WEBP) diff --git a/examples/server/README.md b/examples/server/README.md index 7ead1d9e2..1bb49a59f 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -1,3 +1,248 @@ +# Model selection + +Start without model arguments to select models in the web UI: + +```sh +sd-server --model-dir ./models +``` + +From a source checkout, you can also use `sh ./start-server.sh` on Linux/macOS +or `.\start-server.bat` on Windows. The launchers look for the server relative +to the script: beside the script, then in `bin/Release/`, `bin/`, +`build/bin/Release/`, and `build/bin/`, in that order. +For a different build directory, set `SD_SERVER_PATH` to the executable path: + +```sh +SD_SERVER_PATH="./build_vulkan/bin/sd-server" sh ./start-server.sh --listen-port 8080 +``` + +```powershell +$env:SD_SERVER_PATH = ".\build_vulkan\bin\Release\sd-server.exe" +.\start-server.bat --listen-port 8080 +``` + +Binary release archives include both launchers. Keep the launchers under version +control and put personal settings in a separate file beside them: + +- Windows batch: `start-server.local.bat` +- Linux/macOS: `start-server.local.sh` + +These local files are ignored by Git and excluded from release archives. They +are loaded automatically after the launcher's defaults. They can override +`SD_SERVER_PATH`, `MODEL_DIR`, and `COMMANDLINE_ARGS`; there is no need to edit +the tracked launchers. For example: + +```bat +@echo off +set "MODEL_DIR=D:\stable-diffusion-webui\models" +set "COMMANDLINE_ARGS=--listen-port 8080 --offload-to-cpu" +``` + +```sh +MODEL_DIR="/data/stable-diffusion-webui/models" +COMMANDLINE_ARGS="--listen-port 8080 --offload-to-cpu" +``` + +Each launcher prints the resolved command before execution and forwards the +server's exit code. + +`SD_SERVER_PATH` inherits the environment value by default; leave it empty for +automatic discovery, or set it in the local file to select another executable. +Settings are applied in this order: launcher defaults, local configuration, +then invocation arguments. Repeated `--model-dir` arguments add roots, so they +do not replace the `MODEL_DIR` root. Local configuration files are shell scripts +and use the corresponding shell's syntax. + +The launchers preserve the current working directory, so the default +`MODEL_DIR="./models"` is relative to where you start the launcher. Arguments +passed when invoking the launcher follow these defaults. Repeated `--model-dir` +arguments add search roots rather than replacing `MODEL_DIR`; set `MODEL_DIR` in +the local configuration to change the first root. Other single-value options can override the defaults. +In the shell script, `COMMANDLINE_ARGS` is a +whitespace-separated list; pass values containing spaces as quoted arguments +when invoking the script. `MODEL_DIR` itself supports spaces in both scripts. +A leading `--version` bypasses the configured defaults. An explicit +`SD_SERVER_PATH` is used without falling back to another build if it is invalid. + +Choose a recipe, select the required component files, and click **Load models**. +Recipe and component selectors support searching by name or relative file ID. +Use the arrow keys and Enter to select, or Escape to cancel a search. With +**Apply recipe defaults** enabled, selecting a recipe immediately applies its +declared generation defaults while preserving prompts, input images and fields +not specified by the recipe. Disable it to keep your current parameters when +switching recipes. This preference defaults to enabled and is saved locally. +Model loading and server-info refreshes do not reapply defaults or reset the form. +Recipes declaring `video_frames` also switch the form to Video Generation; +other recipes switch to Image Generation. Generation is enabled after the +selected recipe is loaded. Changing a selection does not reload the model +until you click Load. +The server can also start with the existing model arguments shown below; +existing API clients then work without selecting a recipe. The UI automatically +selects **Command-line model (all features)** and uses the startup generation +defaults. This runtime-only recipe exposes all controls for the supported +image/video mode. Selecting another recipe previews a model switch; selecting +the command-line entry again restores its generation defaults. The entry is +removed when the command-line context is unloaded or replaced. + +Built-in recipes cover SD 1/2/3/3.5, SDXL and distilled checkpoints; +FLUX 1/2/3, Chroma, Qwen Image and editing variants; Z-Image, Anima, ERNIE, +LENS, PiD and other supported image families; AnimateDiff, Wan, LTX, +HunyuanVideo, LingBot Video and MiniMax-H3. Model-specific text encoder +labels identify the expected encoder family and size. Loading a video model enables **Video Generation**; I2V also requires a +start image. The Models panel can be collapsed and remembers its open state. +Its header shows elapsed loading time from the status API and keeps the final +time after success or failure, including after a page refresh. +Built-ins are embedded in the +binary from [recipes/](recipes/), with one JSON file per recipe. Adding or +editing a built-in requires rebuilding the server. User recipes are read from +`/recipes/*.json`. A valid user recipe with the same ID replaces +the entire built-in recipe. Across roots, the first valid user recipe wins; +overrides are logged. Duplicate user IDs within one root are rejected; invalid +recipes are reported without removing a valid recipe from another root or the +built-in version. Slots need only `id` and +`required`; the server supplies labels and search directories. Use `label` +only when a recipe needs a more specific display name. + +Recipes use `features` to enable specialized controls and `disabled_features` +to turn off shared defaults. Disabled features take precedence. The UI hides +unsupported fields, empty sections and irrelevant sampler/cache options, and +omits hidden values from generation requests. See the +[feature declarations](api.md#recipe-feature-declarations) for the defaults +and complete list of supported names. + +Built-in additions and changes must follow the [recipe policy](recipes/README.md). +Split recipes by task weights or added capabilities, not by style or acceleration +LoRAs. Quantizations, file formats and compatible layer-count variants share a +recipe. Native encoders, projectors and connectors stay in the model's recipe. +Community editing recipes require selecting the matching LoRA separately; a +recipe does not enable it automatically. + +See [generation defaults and sources](recipes/defaults.md) for the selected +publisher recommendations and reference-code defaults, with step defaults capped +at 20. Recipes link to the +original publisher's Hugging Face repository where one is available. DiT +recipes default to Euler, except for models requiring a specialized sampler. + +Compatible recipes offer a TAE slot alongside VAE. When the checkpoint does not +bundle a VAE, select at least one of VAE or TAE. The Models panel shows a reminder +below the model selectors until this requirement is met. A selected TAE is used for +encoding and decoding, including when a full VAE is also selected; use complete, +model-compatible TAE weights rather than decoder-only preview weights. Qwen +Image 2.1 uses the same recipe for its full VAE and TAEQI2.1 alternatives. + +The following variants have dedicated recipes: + +| Model | Recipe IDs / notes | +| --- | --- | +| Z-Image L2P | `z-image-l2p`; no VAE | +| LLaDA-Image | `llada-image` (base) and `llada-image-turbo`; use matching encoders and connectors | +| PixArt Alpha | `pixart-alpha-512`, `pixart-alpha-1024`; Alpha VAE scale and positional settings | +| PiD / PiD 1.5 | `pid` / `pid-15` for Flux or Z-Image VAE; append `-sd3`, `-flux2`, or `-qwen-image` for other latent formats | +| SDXL adapters | `sdxl-ip-adapter` supports classic and Plus ViT-H adapters; `sdxl-controlnet` requires compatible SDXL weights | +| FLUX.1 Fill / Depth | `flux1-fill` uses Init Image and Mask Image; `flux1-depth` uses a precomputed depth map in Control Image | +| SD2 / SDXL inpainting | `sd2-inpainting`, `sdxl-inpainting` | +| SD-Turbo | `sd-turbo`; single-file SD2.1-derived checkpoint | +| Community editing LoRAs | `anima-edit`, `krea2-raw-edit`, `krea2-turbo-edit`; apply a matching LoRA after loading | +| HunyuanVideo 1.5 | `hunyuan-video-15-t2v`, `hunyuan-video-15-i2v`; choose the task's checkpoint | +| Wan 2.1 / compatible single Wan 2.2 experts | `wan21-t2v`, `wan21-i2v`; the I2V recipe also accepts compatible SkyReels V2 weights | +| Qwen Image 2.1 Turbo | `qwen-image-21-turbo`; standalone Turbo weights with the fixed 8-step schedule, Euler and CFG 1 | +| MOSS Image | `moss-image-base`, `moss-image-sft`; choose the corresponding checkpoint | + +PiD always requires a reference image and uses the first one. Its recipes expose +reference inputs rather than ordinary image-to-image inputs. IP-Adapter recipes +expose a separate IP-Adapter image and strength. FLUX Canny/Depth consume Control +Image directly without a separate ControlNet; the UI does not calculate edge +or depth maps. Their strength is controlled by guidance, not ControlNet strength. +Krea2 edit recipes use the default `krea2_ostris_edit` reference preset, so the +editing LoRA must match that convention. Ordinary Anima and Krea2 recipes do not +expose reference-image editing inputs. + +Fill/Depth guidance defaults follow the [Diffusers FLUX documentation](https://huggingface.co/docs/diffusers/api/pipelines/flux). +SD-Turbo uses the [model card's one-step configuration](https://huggingface.co/stabilityai/sd-turbo), +with CFG disabled using scale 1 in sd.cpp. + +SVD is not included: legacy model recognition and UNet branches remain, but the +current runner/conditioning setup does not provide a dedicated SVD image-conditioning +path. Adding a recipe alone would not establish a working SVD pipeline. + +Reference images, masks, audio, first/last video frames, and generation options +such as upscaling remain generation inputs. Hardware and memory options remain +startup arguments. Tiny SD/Tiny SD2 and Diffusers LCM directories must first be +converted into single-file checkpoints for the file catalog. LCM checkpoints +and LoRAs use the matching SD family recipe; select LCM sampling and adjust +steps/CFG for the chosen weights. +Metadata inspection, conversion, standalone upscaling and ADetailer are CLI +workflows rather than model-loading recipes. ESRGAN and LTX latent upscalers +are selected through generation upscaling settings with `--hires-upscalers-dir`; +textual inversion uses the startup `--embd-dir` option. + +Repeat `--model-dir` to share multiple model directories, including an existing +SD WebUI model directory: + +```sh +sd-server --model-dir ./models --model-dir /data/stable-diffusion-webui/models +``` + +With no `--model-dir`, the server uses `./models`. Explicit roots replace that +default and are scanned in argument order. Duplicate canonical roots and file +paths are scanned once. Every root can contain a `recipes/` directory. +Directory names are matched case-insensitively on all platforms: + +| Category | Directory names | +| --- | --- | +| Checkpoints | `checkpoints`, `Stable-diffusion` | +| VAE | `vae`, `VAE` | +| LoRA | `loras`, `Lora` | +| ControlNet | `controlnet`, `ControlNet` | + +Other categories keep their existing directory names. Classification uses only +the containing directory: `*.vae.safetensors`, `*.vae.ckpt`, and similar names +inside `checkpoints/` or `Stable-diffusion/` remain checkpoints. + +Place model files in category directories, for example: + +```text +models/ + checkpoints/ + diffusion_models/ + text_encoders/ + vae/ + clip_vision/ + loras/ + recipes/ + my-flux.json +``` + +Subdirectories within a model category are supported. Use **Refresh files & +recipes** after adding files or changing recipes. Scanning lists file names +without loading model weights. Category membership does not guarantee +compatibility; the model loader performs the final compatibility check. +Symlinks resolving outside the model directory are rejected. Dropdown labels +omit the category prefix while retaining subdirectory names. Identical display +paths are displayed as full file paths, with source directories before the +filename instead of a hash suffix. The same applies to duplicate LoRA paths. File IDs include a +root qualifier for additional roots; clients should use the returned IDs. +Empty optional categories do not display missing-file hints. + +Place Tiny AutoEncoder (TAE) files in `vae/` and select them in the **TAE** +slot. The legacy `vae_approx/` directory is also scanned for TAE selections. + +LoRAs are discovered automatically in both `loras/` and `Lora/` under every +model root. Use `--lora-model-dir` to replace automatic discovery with one +explicit LoRA directory. Duplicate names remain separate selectable entries. The LoRA selectors support +searching by name or path with the same keyboard controls as model selectors. + +The server keeps one model context loaded. Loading or unloading is rejected +while generation requests or queued jobs are pending. A switch validates the +selection before unloading the old context, then loads the replacement. +If loading fails, the service stays online with no active model and reports +the error. Backend and memory options supplied at startup apply to subsequent +loads. Recipe defaults apply to new loads; explicitly supplied generation +request parameters take precedence. + +See [the API reference](api.md#model-management) for recipe fields, supported +slots, management endpoints, and compatibility behavior. + # Example The following example starts `sd-server` with a standalone diffusion model, VAE, and LLM text encoder: diff --git a/examples/server/api.md b/examples/server/api.md index cff4e8455..bb4252366 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -59,6 +59,293 @@ Current endpoints include: - `POST /sdcpp/v1/vid_gen` - `POST /sdcpp/v1/upscale` +## Model management + +Model management is additive: existing generation request bodies, successful +responses, job polling and cancellation remain unchanged. All three API families +use the current loaded context. Starting with the existing model arguments +preserves command-line preloading and its generation defaults; it does not +require a recipe. + +Recipe defaults apply when a generation request omits the corresponding +parameters. Image editing endpoints retain their existing input-image size +inference when dimensions are omitted. + +Without a loaded model, generation endpoints return HTTP 503 with +`{"error":"No model is loaded","code":"model_not_loaded"}`. +While loading/unloading, endpoints that access the context return HTTP 503 +with `code: "model_loading"`. These errors retain the existing string-valued +`error` field. Job polling/cancellation, catalog queries and model status remain +available during loading. Model lists are empty when no model is loaded; +`/v1/models` retains the `sd-cpp-local` ID when ready. + +| Method | Endpoint | Response / behavior | +| --- | --- | --- | +| GET | `/sdcpp/v1/recipes` | `{recipes: [...], errors: [...]}` | +| GET | `/sdcpp/v1/model-files` | `{files: [{id, name, category, root, relative_path}], errors: [...]}` | +| POST | `/sdcpp/v1/models/refresh` | Rescan recipes and files; return `{errors: [...]}` | +| GET | `/sdcpp/v1/models/status` | Current state, recipe, selection, error, loading time and optional revision | +| POST | `/sdcpp/v1/models/load` | Load the submitted recipe and selection; return status after completion | +| POST | `/sdcpp/v1/models/unload` | Release the active context; return empty status | + +Load request: + +```json +{ + "recipe": "flux1-dev", + "models": { + "diffusion_model": "diffusion_models/flux1-dev.gguf", + "clip_l": "text_encoders/clip_l.safetensors", + "t5xxl": "text_encoders/t5xxl.safetensors", + "vae": "vae/ae.safetensors" + } +} +``` + +File IDs are opaque strings returned by `model-files`. The first model root +retains the existing relative IDs, such as `checkpoints/nested/model.gguf`. +Additional roots use `@/` so identical paths in different +roots remain selectable. Root qualifiers derive from the canonical root path; +adding unrelated roots or refreshing the catalog does not change them. Moving +a root changes its qualifier. Do not construct IDs or use display names in load +requests. + +The display `name` omits the physical category directory (including aliases such +as `Stable-diffusion`). Duplicate display paths use the full absolute file path, +with the source directories before the filename and no hash suffix. +`root` is an opaque source identifier, and +`relative_path` is the file path beneath that root. Duplicate canonical files +appear only once, using the first root that discovers them. Categories depend +on directories, never on filename suffixes such as `.vae.safetensors`. + +Repeated `--model-dir` arguments merge roots in order; absent arguments default +to `./models`. User recipes override built-ins; the earliest root containing a +valid user recipe wins across roots. Duplicate recipe IDs within one root are +rejected. Directory aliases do not change recipe slot categories. + +LoRA lists retain relative `path` IDs for `loras/` in the first model root and +for an explicit `--lora-model-dir`, and qualify other directories. The additive `display_path` field provides a readable +relative path, or the full absolute file path when needed to distinguish duplicates. +Continue sending `path` in structured LoRA requests. `--lora-model-dir` overrides +automatic discovery across all model roots. +Absolute paths, unknown files, wrong categories and +paths escaping the configured root are rejected. Optional slots can be omitted +or set to an empty string. Load/unload returns HTTP 409 (`model_busy`) while +requests or queued jobs are pending. Invalid configurations return HTTP 400 +(`invalid_model_config`) before unloading the current context. An initialization +failure returns HTTP 422 (`model_load_failed`) and leaves no active context. +The load HTTP request waits for completion; a separate client can poll status +while it runs. Refreshing the catalog never modifies an already loaded context. + +Status uses `state: empty | loading | ready | error`, `recipe` (ID or null), +`models` (slot-to-file-ID mapping), and `error` (string or null). A monotonically +increasing `revision` identifies runtime transitions within this server process. +Command-line preloading keeps `origin: command_line` and `recipe: null` for +compatibility, and adds `runtime_recipe`: a read-only `@command-line` recipe +with all feature declarations enabled. It is only available while that context +is loaded and cannot be submitted to `models/load`. The UI selects it by default +and uses command-line generation defaults, not defaults from a catalog recipe. +Capabilities also include this status as an additive `runtime` field. + +`load_elapsed_seconds` is the server-measured duration in seconds (a number or +null). It updates while a model is loading and freezes on success or failure. +Timing starts after selection validation and includes releasing the previous +model. It resets for each load and is null before any load and during/after +unloading. Command-line preloading reports its completed initialization time. +Poll `/sdcpp/v1/models/status` to display elapsed time, including while loading; +timer updates alone do not change `revision`. + +### Recipe feature declarations + +Recipes may declare `features` and `disabled_features` arrays. The server starts +with shared defaults, enables features declared by component slots and `features`, +then applies `disabled_features` last. A disabled feature always wins. Unknown +names, non-array values and duplicate names within an array are rejected. +Both fields are optional; existing recipes remain valid. + +```json +{ + "features": ["flow_shift", "distilled_guidance"], + "disabled_features": ["clip_skip", "lora"] +} +``` + +Default-enabled features are `img2img`, `mask_image`, `lora`, `vae_tiling`, +`cache`, `cache_dit` and `preview`. All other features below default to false: + +| Feature | Controls | +| --- | --- | +| `flow_shift` | Flow shift, where used by the selected scheduler | +| `distilled_guidance` | Distilled guidance embedding | +| `shifted_timestep` | SDXL timestep shifting | +| `img_cfg` | Image CFG | +| `slg` | Skip-layer guidance fields | +| `clip_skip` | CLIP Skip | +| `img2img`, `mask_image` | Image initialization/strength and masks | +| `ref_images` | Reference image inputs | +| `controlnet` | Control image and ControlNet strength | +| `control_image` | Control image without a separate ControlNet (FLUX Canny/Depth) | +| `ip_adapter` | IP-Adapter image and strength | +| `init_image`, `end_image` | Video start/end frames | +| `control_frames`, `vace` | Video control frames and VACE strength | +| `high_noise` | High-noise pass, MoE boundary and high-noise LoRAs | +| `lora`, `vae_tiling` | LoRA overrides and VAE tiling | +| `cache`, `cache_unet`, `cache_dit` | Cache panel and architecture-specific algorithms | +| `preview`, `preview_tae` | Preview panel and TAE preview option | + +A `control_net` slot enables `controlnet`, `ip_adapter` enables `ip_adapter`, +`high_noise_diffusion_model` enables `high_noise`, and `tae`/`taesd` enables `preview_tae`, unless explicitly disabled. +ControlNet controls are shown when previewing a supporting recipe, even before +a model file is selected. High-noise and TAE controls additionally require their +component to be selected. Runtime capabilities require the component to be loaded. +PuLID disables SLG while selected. +Image-only and video-only inputs are filtered by the active generation mode. +Eta depends on the selected sampler; SCM fields only apply to DBCache, +TaylorSeer and Cache-DIT. Empty panels and advanced sections are hidden. + +`GET /sdcpp/v1/recipes` adds a resolved boolean `effective_features` object to +each recipe. This is response-only metadata, like `source` and slot `categories`; +omit it when writing a user recipe file. The defaults are maintained by the +server, not duplicated by the frontend. + +`GET /sdcpp/v1/capabilities` adds `generation_features` for the loaded context, +derived from the active recipe and component availability without inferring +features from model-family names. When no model is loaded, every feature is +present and false. The UI can still preview a selected recipe using its +`effective_features` before loading. +Existing `features` / `features_by_mode` keys are retained and their model-specific +values reflect these declarations. A loaded recipe is snapshotted: refreshing +recipe files does not change the active context's capabilities until reloading. +Command-line preloading uses the runtime-only all-features recipe instead of model-family inference +and component paths; variant-specific recipe declarations are more precise. + +These declarations govern the Web UI and advertised capabilities, not API +request validation. Existing generation APIs continue accepting their existing +fields. The Web UI omits unsupported/hidden fields and ignores hidden image +inputs when building requests, so switching recipes cannot submit stale options. + +### User recipe format + +Contributions to the built-in catalog must follow the +[built-in recipe policy](recipes/README.md). Personal user recipes may define +workflow-specific presets without adding them to the built-in catalog. + +Each UTF-8 JSON file in `/recipes/` defines one complete recipe. +The maximum recipe file size is 1 MiB. Unknown fields, invalid types and +duplicate slots are errors. Built-ins in `examples/server/recipes/*.json` use +the same one-recipe-per-file schema: + +```json +{ + "schema_version": 1, + "id": "my-flux", + "name": "My FLUX", + "description": "FLUX.1 dev", + "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-dev", + "slots": [ + {"id": "diffusion_model", "required": true}, + {"id": "clip_l", "required": true}, + {"id": "t5xxl", "required": true}, + {"id": "vae", "required": false}, + {"id": "tae", "required": false} + ], + "required_any": [["vae", "tae"]], + "defaults": {"width": 1024, "height": 1024, "cfg_scale": 1.0, "distilled_guidance": 3.5, "sampling_method": "euler", "steps": 20} +} +``` + +IDs contain 1–128 ASCII letters, digits, dots, underscores or hyphens. +`name`, `schema_version: 1` and `slots` are required; `description` and +`model_url`, `required_any`, `defaults`, `context`, `features` and `disabled_features` are optional. Each slot requires only `id` and boolean `required`. +The server supplies the default label and search directories for each slot. +An optional nonempty `label` overrides the display name, for example +`{"id": "diffusion_model", "label": "Low-noise diffusion model", "required": true}`. +Legacy `category` fields remain accepted, but must match one of the slot's +supported directories; they cannot redefine or restrict its search directories. +At least one required `model` or `diffusion_model` slot is necessary. + +`model_url` is an optional HTTPS Hugging Face model repository link, shown beside +the recipe description. Built-ins link to the model's author/publisher, not repacks. +`required_any` contains groups of declared slot IDs. At least one model in each +group must be selected. For example, `"required_any": [["vae", "tae"]]` with both +slots marked `required: false` allows either the full VAE or a compatible TAE. +Selecting both is allowed; the current server uses TAE for encoding/decoding when +TAE is selected. Checkpoint recipes with a bundled VAE do not need this group. +Use complete, model-compatible TAE weights; decoder-only preview weights cannot +replace a full VAE. Pixel-space models and unsupported latent formats have no +TAE slot. + +| Slot IDs | Search directories | +| --- | --- | +| `model` | `checkpoints` | +| `diffusion_model`, `high_noise_diffusion_model`, `uncond_diffusion_model` | `diffusion_models` | +| `clip_l`, `clip_g`, `t5xxl`, `llm`, `audio_encoder`, `embeddings_connectors` | `text_encoders` | +| `clip_vision`, `llm_vision` | `clip_vision` | +| `vae`, `audio_vae` | `vae` | +| `tae` (legacy alias: `taesd`) | `vae` or `vae_approx` | +| `tokenizer` | `tokenizers` | +| `control_net` | `controlnet` | +| `ip_adapter` | `ipadapter` | +| `motion_module` | `motion_modules` | +| `photo_maker` | `photomaker` | +| `pulid_weights` | `pulid` | + +Built-in recipes use `tae` for the Tiny AutoEncoder slot. TAE selections can use files from either `vae/` or the legacy +`vae_approx/` directory. Existing recipes and load requests using `taesd` +remain supported. Do not specify both `tae` and `taesd` in the same recipe +or load request. + +Model files can use `.gguf`, `.safetensors`, `.sft`, `.ckpt`, `.pt`, `.pth` +or `.bin` extensions. The tokenizer category lists `.json` files only. +Custom recipes compose components supported by the backend; they do not add +new model architectures. + +Optional `context` settings apply to each model load and override the matching +startup setting. Supported fields are `vae_format` (`auto`, `flux`, `sd3`, +`flux2`, or `wan`) and `model_args` (a string in the same format as +`--model-args`; replaces the startup string). Other startup settings remain +unchanged. Switching to a recipe that omits these fields restores the startup +values. For example, PiD uses `{"vae_format": "flux"}` and Qwen Image Edit +2511 uses `{"model_args": "qwen_image_zero_cond_t=true"}`. + +Supported defaults: + +- Integer `width`/`height` (64–4096), `steps` (1–1000), `video_frames` + (1–4096), `fps` (1–240), `shifted_timestep` (0–1000), and + `high_noise_steps` (1–1000 or -1 for automatic). +- Numeric `cfg_scale`, `img_cfg_scale`, `distilled_guidance`, `flow_shift`, + `high_noise_cfg_scale`, and `high_noise_flow_shift` (0–1000). +- Numeric `strength` (0–1), used for image-to-image denoising, and + `moe_boundary` (0–1), used for automatic Wan MoE expert switching. +- `custom_sigmas`: an empty array or 2–1001 finite, nonnegative numbers. + A nonempty array overrides the generated schedule; the sigma count minus one + determines the actual step count. Include the final zero. An empty array uses + the generated schedule. Loaded recipe sigmas apply to generation requests + unless `sample_params.custom_sigmas` overrides them; switching recipes restores + that recipe's schedule or the startup default. +- Supported `sampling_method`/`scheduler` and + `high_noise_sampling_method`/`high_noise_scheduler` names. + +Other generation parameters continue to use startup defaults unless explicitly +supplied by the generation request. Built-in Wan recipes provide video defaults; +use `/sdcpp/v1/vid_gen` after loading a video model. +LoRAs are merged from `loras/` and `Lora/` in every model root unless +`--lora-model-dir` is supplied. + +Recipe responses fill in each slot's `label`, `category` (the legacy value, +if supplied, or the first search directory), and `categories` (the complete +server-defined list of search directories). Clients should filter model files +using `categories`; `category` remains available for older clients. +For example, `{"id": "tae", "required": false}` expands to +`{"id": "tae", "required": false, "label": "TAE", "category": "vae", "categories": ["vae", "vae_approx"]}`. + +Recipe responses also add `source: builtin | user` and, for user recipes, `file`. +Do not include `source`, `file`, or slot `categories` in a recipe file. A valid user recipe +replaces a built-in with the same ID in full; inheritance/partial merging is +not supported. Invalid recipes are skipped. Multiple valid user files sharing +an ID are all skipped, preserving any built-in with that ID. Catalog errors +are returned as `{file, message}` entries. + ## `sd_cpp_extra_args` `sd_cpp_extra_args` is an extension mechanism for the compatibility APIs. @@ -155,6 +442,10 @@ image establishes the generation dimensions if `size` is omitted. Input geometry follows `image_preprocess`: references preserve their dimensions by default, while init and mask use the generation canvas preset. +When the request starts without preconfigured reference images, the first uploaded image becomes the img2img init image if the loaded model does not support reference-image conditioning, the final reference list is empty, a mask is present, or an explicit numeric `strength` override is provided. Otherwise, only the final reference list is used for image conditioning, without initializing the img2img latent. Requests with preconfigured references retain their existing init-image selection. + +Routing uses the reference list and mask after applying `sd_cpp_extra_args`. Both `ref_images: []` and `ref_images: null` clear the reference list; `mask_image: null` clears an uploaded mask. An explicit `init_image` in `sd_cpp_extra_args`, including `null`, takes precedence over the implicit init-image selection. + Reference encoding then follows model presets and `ref_image_args`. To skip input geometry for references and disable resizing before VAE encoding, include this in `prompt`: @@ -409,7 +700,7 @@ Common job shape: "id": "job_01HTXYZABC", "kind": "img_gen", "status": "queued", - "created": 1775401200, + "created": 1775401200000, "started": null, "completed": null, "queue_position": 2, @@ -433,6 +724,11 @@ Field types: | `error` | `object \| null` | | `preview` | `object \| null` | +`created`, `started`, and `completed` are integer Unix timestamps in +**milliseconds**. Subtract `started` from `completed` and divide by 1000 to get +the elapsed generation time in seconds, including output encoding and excluding +queue time. + `preview` sub-fields: | Field | Type | Notes | @@ -508,6 +804,7 @@ Shared nested fields: | --- | --- | | `loras[].name` | `string` | | `loras[].path` | `string` | +| `loras[].display_path` | `string` (display only) | `upscalers` @@ -662,7 +959,7 @@ Example response: "id": "job_01HTXYZABC", "kind": "img_gen", "status": "queued", - "created": 1775401200, + "created": 1775401200000, "poll_url": "/sdcpp/v1/jobs/job_01HTXYZABC" } ``` @@ -990,9 +1287,9 @@ Example completed job: "id": "job_01HTXYZABC", "kind": "img_gen", "status": "completed", - "created": 1775401200, - "started": 1775401203, - "completed": 1775401215, + "created": 1775401200000, + "started": 1775401203000, + "completed": 1775401215237, "queue_position": 0, "result": { "output_format": "png", @@ -1016,9 +1313,9 @@ Example failed job: "id": "job_01HTXYZABC", "kind": "img_gen", "status": "failed", - "created": 1775401200, - "started": 1775401203, - "completed": 1775401204, + "created": 1775401200000, + "started": 1775401203000, + "completed": 1775401204000, "queue_position": 0, "result": null, "error": { @@ -1037,9 +1334,9 @@ Example cancelled job: "id": "job_01HTXYZABC", "kind": "img_gen", "status": "cancelled", - "created": 1775401200, + "created": 1775401200000, "started": null, - "completed": 1775401202, + "completed": 1775401202000, "queue_position": 0, "result": null, "error": { @@ -1075,7 +1372,7 @@ Example response: "id": "job_01HTXYZVID", "kind": "vid_gen", "status": "queued", - "created": 1775401200, + "created": 1775401200000, "poll_url": "/sdcpp/v1/jobs/job_01HTXYZVID" } ``` @@ -1403,9 +1700,9 @@ Example completed job: "id": "job_01HTXYZVID", "kind": "vid_gen", "status": "completed", - "created": 1775401200, - "started": 1775401203, - "completed": 1775401215, + "created": 1775401200000, + "started": 1775401203000, + "completed": 1775401215237, "queue_position": 0, "result": { "output_format": "webm", @@ -1429,9 +1726,9 @@ Example failed job: "id": "job_01HTXYZVID", "kind": "vid_gen", "status": "failed", - "created": 1775401200, - "started": 1775401203, - "completed": 1775401204, + "created": 1775401200000, + "started": 1775401203000, + "completed": 1775401204000, "queue_position": 0, "result": null, "error": { @@ -1450,9 +1747,9 @@ Example cancelled job: "id": "job_01HTXYZVID", "kind": "vid_gen", "status": "cancelled", - "created": 1775401200, + "created": 1775401200000, "started": null, - "completed": 1775401202, + "completed": 1775401202000, "queue_position": 0, "result": null, "error": { diff --git a/examples/server/async_jobs.cpp b/examples/server/async_jobs.cpp index 88408088e..7f3c8a1bf 100644 --- a/examples/server/async_jobs.cpp +++ b/examples/server/async_jobs.cpp @@ -88,7 +88,7 @@ const char* async_job_status_name(AsyncJobStatus status) { } void purge_expired_jobs(AsyncJobManager& manager) { - const int64_t now = unix_timestamp_now(); + const int64_t now = unix_timestamp_now_ms(); for (auto it = manager.expired_jobs.begin(); it != manager.expired_jobs.end();) { if (it->second <= now) { @@ -108,8 +108,8 @@ void purge_expired_jobs(AsyncJobManager& manager) { int64_t ttl_seconds = job->status == AsyncJobStatus::Completed ? manager.completed_ttl_seconds : manager.failed_ttl_seconds; - if (now - job->completed_at >= ttl_seconds) { - manager.expired_jobs[job->id] = now + std::max(ttl_seconds, 60); + if (now - job->completed_at >= ttl_seconds * 1000) { + manager.expired_jobs[job->id] = now + std::max(ttl_seconds, 60) * 1000; it = manager.jobs.erase(it); } else { ++it; @@ -130,7 +130,7 @@ size_t count_pending_jobs(const AsyncJobManager& manager) { std::string make_async_job_id(AsyncJobManager& manager) { std::ostringstream oss; - oss << "job_" << std::hex << unix_timestamp_now() << "_" << std::setw(8) + oss << "job_" << std::hex << unix_timestamp_now_ms() << "_" << std::setw(8) << std::setfill('0') << manager.next_id++; return oss.str(); } @@ -143,7 +143,7 @@ bool cancel_queued_job(AsyncJobManager& manager, AsyncGenerationJob& job) { manager.queue.erase(new_end, manager.queue.end()); job.status = AsyncJobStatus::Cancelled; - job.completed_at = unix_timestamp_now(); + job.completed_at = unix_timestamp_now_ms(); job.result_images_b64.clear(); job.result_media_b64.clear(); job.result_media_mime_type.clear(); @@ -373,7 +373,7 @@ void async_job_worker(ServerRuntime& runtime) { job = it->second; job->status = AsyncJobStatus::Generating; - job->started_at = unix_timestamp_now(); + job->started_at = unix_timestamp_now_ms(); } std::vector output_images; @@ -405,7 +405,7 @@ void async_job_worker(ServerRuntime& runtime) { continue; } - job->completed_at = unix_timestamp_now(); + job->completed_at = unix_timestamp_now_ms(); if (ok) { job->status = AsyncJobStatus::Completed; job->result_images_b64 = std::move(output_images); diff --git a/examples/server/async_jobs.h b/examples/server/async_jobs.h index 2fe79bb24..049a9d73b 100644 --- a/examples/server/async_jobs.h +++ b/examples/server/async_jobs.h @@ -32,7 +32,7 @@ struct AsyncGenerationJob { std::string id; AsyncJobKind kind = AsyncJobKind::ImgGen; AsyncJobStatus status = AsyncJobStatus::Queued; - int64_t created_at = unix_timestamp_now(); + int64_t created_at = unix_timestamp_now_ms(); int64_t started_at = 0; int64_t completed_at = 0; ImgGenJobRequest img_gen; diff --git a/examples/server/frontend b/examples/server/frontend index 8e7c11505..e36e34a2e 160000 --- a/examples/server/frontend +++ b/examples/server/frontend @@ -1 +1 @@ -Subproject commit 8e7c11505925c0dbf9aaea56a3a5058b8690d66b +Subproject commit e36e34a2e2204e3df654b16cadc112d8ce8f3174 diff --git a/examples/server/main.cpp b/examples/server/main.cpp index ddfd7334e..170877686 100644 --- a/examples/server/main.cpp +++ b/examples/server/main.cpp @@ -10,6 +10,7 @@ #include "async_jobs.h" #include "common/common.h" #include "common/resource_owners.hpp" +#include "model_catalog.h" #include "routes.h" #include "runtime.h" @@ -50,7 +51,7 @@ static void parse_args(int argc, const bool random_seed_requested = default_gen_params.seed < 0; if (!svr_params.resolve_and_validate() || - !ctx_params.resolve_and_validate(IMG_GEN) || + !ctx_params.resolve_and_validate(ctx_params.model_path.empty() && ctx_params.diffusion_model_path.empty() ? METADATA : IMG_GEN) || !default_gen_params.resolve_and_validate(IMG_GEN, ctx_params.lora_model_dir, ctx_params.hires_upscalers_dir)) { @@ -84,14 +85,19 @@ int main(int argc, const char** argv) { LOG_VERBOSE("version: %s", version_string().c_str()); LOG_VERBOSE("%s", sd_get_system_info()); - LOG_VERBOSE("%s", svr_params.to_string().c_str()); + LOG_DEBUG("%s", svr_params.to_string().c_str()); LOG_DEBUG("%s", ctx_params.to_string().c_str()); LOG_DEBUG("%s", default_gen_params.to_string().c_str()); sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t(false); - SDCtxPtr sd_ctx(new_sd_ctx(&sd_ctx_params)); + ServerModels models; + if (!ctx_params.model_path.empty() || !ctx_params.diffusion_model_path.empty()) { + const auto started = std::chrono::steady_clock::now(); + models.context.reset(new_sd_ctx(&sd_ctx_params)); + models.status["load_elapsed_seconds"] = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + } - if (sd_ctx == nullptr) { + if ((!ctx_params.model_path.empty() || !ctx_params.diffusion_model_path.empty()) && !models.context) { LOG_ERROR("new_sd_ctx_t failed"); return 1; } @@ -104,7 +110,7 @@ int main(int argc, const char** argv) { std::mutex upscaler_mutex; AsyncJobManager async_job_manager; ServerRuntime runtime = { - sd_ctx.get(), + models.context.get(), &sd_ctx_mutex, &svr_params, &ctx_params, @@ -114,8 +120,16 @@ int main(int argc, const char** argv) { &upscaler_cache, &upscaler_mutex, &async_job_manager, + &models, }; + try { + initialize_server_models(runtime); + } catch (const std::exception& e) { + LOG_ERROR("model catalog initialization failed: %s", e.what()); + return 1; + } + std::thread async_worker(async_job_worker, std::ref(runtime)); httplib::Server svr; @@ -147,6 +161,7 @@ int main(int argc, const char** argv) { register_openai_api_endpoints(svr, runtime); register_sdapi_endpoints(svr, runtime); register_sdcpp_api_endpoints(svr, runtime); + register_model_endpoints(svr, runtime); LOG_INFO("listening on: http://%s:%d\n", svr_params.listen_ip.c_str(), svr_params.listen_port); svr.listen(svr_params.listen_ip, svr_params.listen_port); diff --git a/examples/server/model_catalog.cpp b/examples/server/model_catalog.cpp new file mode 100644 index 000000000..a3c59206d --- /dev/null +++ b/examples/server/model_catalog.cpp @@ -0,0 +1,664 @@ +#include "model_catalog.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/log.h" +#include "recipes_builtin.h" + +namespace fs = std::filesystem; + +struct ModelSlot { + const char* id; + const char* label; + std::vector categories; + std::string SDContextParams::*field; +}; + +static const ModelSlot k_model_slots[] = { + {"model", "Checkpoint", {"checkpoints"}, &SDContextParams::model_path}, + {"diffusion_model", "Diffusion model", {"diffusion_models"}, &SDContextParams::diffusion_model_path}, + {"high_noise_diffusion_model", "High-noise diffusion model", {"diffusion_models"}, &SDContextParams::high_noise_diffusion_model_path}, + {"uncond_diffusion_model", "Unconditional diffusion model", {"diffusion_models"}, &SDContextParams::uncond_diffusion_model_path}, + {"clip_l", "CLIP-L", {"text_encoders"}, &SDContextParams::clip_l_path}, + {"clip_g", "CLIP-G", {"text_encoders"}, &SDContextParams::clip_g_path}, + {"clip_vision", "CLIP Vision", {"clip_vision"}, &SDContextParams::clip_vision_path}, + {"t5xxl", "T5", {"text_encoders"}, &SDContextParams::t5xxl_path}, + {"llm", "LLM Text Encoder", {"text_encoders"}, &SDContextParams::llm_path}, + {"llm_vision", "LLM Vision Encoder", {"clip_vision"}, &SDContextParams::llm_vision_path}, + {"tokenizer", "Tokenizer", {"tokenizers"}, &SDContextParams::tokenizer}, + {"vae", "VAE", {"vae"}, &SDContextParams::vae_path}, + {"audio_vae", "Audio VAE", {"vae"}, &SDContextParams::audio_vae_path}, + {"audio_encoder", "Audio Encoder", {"text_encoders"}, &SDContextParams::audio_encoder_path}, + {"embeddings_connectors", "Embeddings Connectors", {"text_encoders"}, &SDContextParams::embeddings_connectors_path}, + {"tae", "TAE", {"vae", "vae_approx"}, &SDContextParams::taesd_path}, + {"taesd", "TAE", {"vae_approx", "vae"}, &SDContextParams::taesd_path}, + {"control_net", "ControlNet", {"controlnet"}, &SDContextParams::control_net_path}, + {"ip_adapter", "IP-Adapter", {"ipadapter"}, &SDContextParams::ip_adapter_path}, + {"motion_module", "Motion Module", {"motion_modules"}, &SDContextParams::motion_module_path}, + {"photo_maker", "PhotoMaker", {"photomaker"}, &SDContextParams::photo_maker_path}, + {"pulid_weights", "PuLID", {"pulid"}, &SDContextParams::pulid_weights_path}, +}; + +static const ModelSlot* find_model_slot(const std::string& id) { + for (const auto& slot : k_model_slots) { + if (id == slot.id) { + return &slot; + } + } + return nullptr; +} + +static bool matches_model_category(const ModelSlot& slot, const std::string& category) { + return std::find(slot.categories.begin(), slot.categories.end(), category) != slot.categories.end(); +} + +static void clear_model_paths(SDContextParams& params) { + for (const auto& slot : k_model_slots) { + (params.*slot.field).clear(); + } +} + +std::string model_path_key(const fs::path& path) { +#ifdef _WIN32 + const auto value = path.generic_wstring(); + const int length = LCMapStringEx(LOCALE_NAME_INVARIANT, LCMAP_LOWERCASE, value.data(), + static_cast(value.size()), nullptr, 0, nullptr, nullptr, 0); + if (length > 0) { + std::wstring folded(length, L'\0'); + LCMapStringEx(LOCALE_NAME_INVARIANT, LCMAP_LOWERCASE, value.data(), + static_cast(value.size()), &folded[0], length, nullptr, nullptr, 0); + return fs::path(folded).generic_u8string(); + } +#endif + return path.generic_u8string(); +} + +bool model_path_inside(const fs::path& root, const fs::path& path) { + auto parent = root.begin(); + auto child = path.begin(); + for (; parent != root.end(); ++parent, ++child) { + if (child == path.end() || model_path_key(*parent) != model_path_key(*child)) { + return false; + } + } + return child != path.end(); +} + +ModelRoot make_model_root(const fs::path& path, bool primary) { + ModelRoot root; + const auto absolute = fs::absolute(path); + std::error_code ec; + root.path = fs::weakly_canonical(absolute, ec); + if (ec) + root.path = absolute.lexically_normal(); + root.primary = primary; + uint64_t hash = 14695981039346656037ULL; + for (unsigned char c : model_path_key(root.path)) { + hash ^= c; + hash *= 1099511628211ULL; + } + std::ostringstream id; + id << '@' << std::hex << std::setfill('0') << std::setw(16) << hash; + root.id = id.str(); + root.label = root.path.filename().u8string() + " [" + root.id.substr(1, 8) + "]"; + return root; +} + +std::vector get_model_roots(const SDSvrParams& params) { + const auto directories = params.model_dirs.empty() ? std::vector{"models"} : params.model_dirs; + std::vector roots; + std::set seen; + for (size_t i = 0; i < directories.size(); ++i) { + auto root = make_model_root(fs::u8path(directories[i]), i == 0); + if (seen.insert(model_path_key(root.path)).second) + roots.push_back(std::move(root)); + } + return roots; +} + +std::vector model_category_directories(const ModelRoot& root, const std::string& category) { + std::vector directories; + if (!fs::exists(root.path)) + return directories; + for (const auto& entry : fs::directory_iterator(root.path)) { + auto name = entry.path().filename().u8string(); + std::transform(name.begin(), name.end(), name.begin(), [](unsigned char c) { return std::tolower(c); }); + if (name == category || (category == "checkpoints" && name == "stable-diffusion") || + (category == "loras" && name == "lora")) { + directories.push_back(entry.path()); + } + } + std::sort(directories.begin(), directories.end(), [&](const fs::path& a, const fs::path& b) { + const bool a_standard = a.filename().u8string() == category; + const bool b_standard = b.filename().u8string() == category; + return a_standard != b_standard ? a_standard : a < b; + }); + return directories; +} + +static bool valid_identifier(const std::string& id) { + return !id.empty() && id.size() <= 128 && + std::all_of(id.begin(), id.end(), [](unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.'; + }); +} + +static void check_keys(const json& object, const std::set& keys) { + if (!object.is_object()) { + throw std::runtime_error("expected an object"); + } + for (auto it = object.begin(); it != object.end(); ++it) { + if (!keys.count(it.key())) { + throw std::runtime_error("unknown field: " + it.key()); + } + } +} + +static void apply_recipe_defaults(const json& defaults, SDGenerationParams& params) { + check_keys(defaults, {"width", "height", "steps", "cfg_scale", "strength", "distilled_guidance", "sampling_method", "scheduler", "custom_sigmas", + "video_frames", "fps", "flow_shift", "moe_boundary", "high_noise_steps", "high_noise_cfg_scale", + "high_noise_flow_shift", "high_noise_sampling_method", "high_noise_scheduler", "img_cfg_scale", "shifted_timestep"}); + for (const char* key : {"width", "height", "steps", "video_frames", "fps", "high_noise_steps", "shifted_timestep"}) { + if (!defaults.contains(key)) { + continue; + } + if (!defaults[key].is_number_integer()) { + throw std::runtime_error(std::string(key) + " must be an integer"); + } + const int64_t value = defaults[key].get(); + const std::string field = key; + const bool dimension = field == "width" || field == "height"; + const int64_t maximum = dimension || field == "video_frames" ? 4096 : field == "fps" ? 240 + : 1000; + const int64_t minimum = dimension ? 64 : field == "shifted_timestep" ? 0 + : 1; + if ((value < minimum || value > maximum) && !(field == "high_noise_steps" && value == -1)) { + throw std::runtime_error(std::string(key) + " is out of range"); + } + if (std::string(key) == "width") + params.width = static_cast(value); + if (std::string(key) == "height") + params.height = static_cast(value); + if (std::string(key) == "steps") + params.sample_params.sample_steps = static_cast(value); + if (field == "shifted_timestep") + params.sample_params.shifted_timestep = static_cast(value); + if (field == "video_frames") + params.video_frames = static_cast(value); + if (field == "fps") + params.fps = static_cast(value); + if (field == "high_noise_steps") + params.high_noise_sample_params.sample_steps = static_cast(value); + } + for (const char* key : {"cfg_scale", "strength", "moe_boundary", "img_cfg_scale", "distilled_guidance", "flow_shift", "high_noise_cfg_scale", "high_noise_flow_shift"}) { + if (!defaults.contains(key)) { + continue; + } + if (!defaults[key].is_number()) { + throw std::runtime_error(std::string(key) + " must be a number"); + } + const float value = defaults[key].get(); + if (!std::isfinite(value) || value < 0 || value > ((std::string(key) == "strength" || std::string(key) == "moe_boundary") ? 1.f : 1000.f)) { + throw std::runtime_error(std::string(key) + " is out of range"); + } + if (std::string(key) == "strength") + params.strength = value; + if (std::string(key) == "moe_boundary") + params.moe_boundary = value; + if (std::string(key) == "cfg_scale") + params.sample_params.guidance.txt_cfg = value; + if (std::string(key) == "img_cfg_scale") + params.sample_params.guidance.img_cfg = value; + if (std::string(key) == "distilled_guidance") + params.sample_params.guidance.distilled_guidance = value; + if (std::string(key) == "flow_shift") + params.sample_params.flow_shift = value; + if (std::string(key) == "high_noise_cfg_scale") + params.high_noise_sample_params.guidance.txt_cfg = value; + if (std::string(key) == "high_noise_flow_shift") + params.high_noise_sample_params.flow_shift = value; + } + if (defaults.contains("custom_sigmas")) { + const auto& sigmas = defaults.at("custom_sigmas"); + if (!sigmas.is_array() || sigmas.size() == 1 || sigmas.size() > 1001) + throw std::runtime_error("custom_sigmas must be empty or contain 2-1001 values"); + for (const auto& sigma : sigmas) { + if (!sigma.is_number() || !std::isfinite(sigma.get()) || sigma.get() < 0) + throw std::runtime_error("custom_sigmas must contain finite nonnegative numbers"); + } + params.custom_sigmas = sigmas.get>(); + } + for (const char* key : {"sampling_method", "high_noise_sampling_method"}) { + if (!defaults.contains(key)) + continue; + auto& sample = std::string(key) == "sampling_method" ? params.sample_params : params.high_noise_sample_params; + const auto name = defaults.at(key).get(); + bool found = false; + for (int i = 0; i < SAMPLE_METHOD_COUNT; ++i) { + if (name == sd_sample_method_name(static_cast(i))) { + sample.sample_method = static_cast(i); + found = true; + break; + } + } + if (!found) + throw std::runtime_error(std::string("unknown ") + key); + } + for (const char* key : {"scheduler", "high_noise_scheduler"}) { + if (!defaults.contains(key)) + continue; + auto& sample = std::string(key) == "scheduler" ? params.sample_params : params.high_noise_sample_params; + const auto name = defaults.at(key).get(); + bool found = false; + for (int i = 0; i < SCHEDULER_COUNT; ++i) { + if (name == sd_scheduler_name(static_cast(i)) || + (name == "normal" && i == DISCRETE_SCHEDULER)) { + sample.scheduler = static_cast(i); + found = true; + break; + } + } + if (!found) + throw std::runtime_error(std::string("unknown ") + key); + } +} + +static void apply_recipe_context(const json& settings, SDContextParams& params) { + check_keys(settings, {"vae_format", "model_args"}); + if (settings.contains("vae_format")) { + const auto format = settings.at("vae_format").get(); + if (format != "auto" && format != "flux" && format != "sd3" && format != "flux2" && format != "wan") + throw std::runtime_error("unknown vae_format"); + params.vae_format = format; + } + if (settings.contains("model_args")) { + params.model_args = settings.at("model_args").get(); + } +} + +static const std::map k_recipe_features = { + {"img2img", true}, + {"mask_image", true}, + {"lora", true}, + {"vae_tiling", true}, + {"cache", true}, + {"cache_dit", true}, + {"preview", true}, + {"flow_shift", false}, + {"distilled_guidance", false}, + {"shifted_timestep", false}, + {"img_cfg", false}, + {"slg", false}, + {"clip_skip", false}, + {"ref_images", false}, + {"controlnet", false}, + {"control_image", false}, + {"ip_adapter", false}, + {"init_image", false}, + {"end_image", false}, + {"control_frames", false}, + {"vace", false}, + {"high_noise", false}, + {"cache_unet", false}, + {"preview_tae", false}, +}; + +static void validate_recipe_features(const json& recipe) { + for (const char* key : {"features", "disabled_features"}) { + if (!recipe.contains(key)) + continue; + const auto& values = recipe.at(key); + if (!values.is_array() || values.size() > k_recipe_features.size()) + throw std::runtime_error(std::string(key) + " must be an array of feature names"); + std::set seen; + for (const auto& value : values) { + const auto name = value.get(); + if (!k_recipe_features.count(name) || !seen.insert(name).second) + throw std::runtime_error("unknown or duplicate feature: " + name); + } + } +} + +json resolve_recipe_features(const json& recipe) { + json features = k_recipe_features; + for (const auto& slot : recipe.at("slots")) { + const auto id = slot.at("id").get(); + if (id == "ip_adapter") + features["ip_adapter"] = true; + if (id == "control_net") + features["controlnet"] = true; + if (id == "high_noise_diffusion_model") + features["high_noise"] = true; + if (id == "tae" || id == "taesd") + features["preview_tae"] = true; + } + for (const auto& name : recipe.value("features", json::array())) + features[name.get()] = true; + for (const auto& name : recipe.value("disabled_features", json::array())) + features[name.get()] = false; + return features; +} + +json model_generation_features(const ServerRuntime& runtime) { + if (!runtime.sd_ctx || runtime.models->active_recipe.is_null()) { + json features = k_recipe_features; + for (auto& feature : features) + feature = false; + return features; + } + json features = resolve_recipe_features(runtime.models->active_recipe); + if (runtime.models->active_recipe.value("source", "") == "command_line") + return features; + // Optional components only become usable once selected and loaded. + features["ip_adapter"] = features.value("ip_adapter", false) && !runtime.ctx_params->ip_adapter_path.empty(); + features["controlnet"] = features.value("controlnet", false) && !runtime.ctx_params->control_net_path.empty(); + features["high_noise"] = features.value("high_noise", false) && !runtime.ctx_params->high_noise_diffusion_model_path.empty(); + features["preview_tae"] = features.value("preview_tae", false) && !runtime.ctx_params->taesd_path.empty(); + if (!runtime.ctx_params->pulid_weights_path.empty()) + features["slg"] = false; + return features; +} + +static void validate_recipe(const json& recipe) { + check_keys(recipe, {"schema_version", "id", "name", "description", "model_url", "slots", "required_any", "defaults", "context", "features", "disabled_features"}); + if (!recipe.at("schema_version").is_number_integer() || recipe.at("schema_version") != 1 || + !valid_identifier(recipe.at("id").get()) || recipe.at("name").get().empty()) { + throw std::runtime_error("invalid schema_version, id or name"); + } + if (recipe.contains("description") && !recipe["description"].is_string()) { + throw std::runtime_error("description must be a string"); + } + if (recipe.contains("model_url")) { + const auto url = recipe.at("model_url").get(); + const std::string prefix = "https://huggingface.co/"; + if (url.size() <= prefix.size() || url.size() > 2048 || url.rfind(prefix, 0) != 0 || + std::any_of(url.begin(), url.end(), [](unsigned char c) { return std::isspace(c) || std::iscntrl(c); })) { + throw std::runtime_error("model_url must be a Hugging Face HTTPS URL"); + } + } + const auto& slots = recipe.at("slots"); + if (!slots.is_array() || slots.empty() || slots.size() > 32) { + throw std::runtime_error("slots must be a nonempty array of at most 32 entries"); + } + std::set ids; + bool has_required_model = false; + for (const auto& slot : slots) { + check_keys(slot, {"id", "label", "category", "required"}); + const auto id = slot.at("id").get(); + const auto* spec = find_model_slot(id); + if (!spec || !ids.insert(id == "taesd" ? "tae" : id).second || + (slot.contains("category") && !matches_model_category(*spec, slot.at("category").get())) || + (slot.contains("label") && slot.at("label").get().empty()) || !slot.at("required").is_boolean()) { + throw std::runtime_error("invalid or duplicate model slot: " + id); + } + has_required_model |= (id == "model" || id == "diffusion_model") && slot["required"].get(); + } + if (!has_required_model) { + throw std::runtime_error("a required model or diffusion_model slot is needed"); + } + if (recipe.contains("required_any")) { + const auto& groups = recipe.at("required_any"); + if (!groups.is_array() || groups.size() > 32) + throw std::runtime_error("required_any must be an array of at most 32 slot groups"); + for (const auto& group : groups) { + if (!group.is_array() || group.size() < 2 || group.size() > slots.size()) + throw std::runtime_error("required_any groups need at least two declared slots"); + std::set seen; + for (const auto& value : group) { + const auto id = value.get(); + if (!seen.insert(id).second || std::none_of(slots.begin(), slots.end(), [&](const json& slot) { return slot.at("id") == id; })) + throw std::runtime_error("unknown or duplicate required_any slot: " + id); + } + } + } + validate_recipe_features(recipe); + SDContextParams context; + apply_recipe_context(recipe.value("context", json::object()), context); + SDGenerationParams defaults; + apply_recipe_defaults(recipe.value("defaults", json::object()), defaults); +} + +static json normalize_recipe(json recipe) { + validate_recipe(recipe); + recipe["effective_features"] = resolve_recipe_features(recipe); + for (auto& slot : recipe.at("slots")) { + const auto& spec = *find_model_slot(slot.at("id").get()); + slot["label"] = slot.value("label", std::string(spec.label)); + slot["category"] = slot.value("category", spec.categories.front()); + slot["categories"] = spec.categories; + } + return recipe; +} + +static bool supported_model_file(const fs::path& path, const std::string& category) { + std::string ext = path.extension().u8string(); + std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return std::tolower(c); }); + if (category == "tokenizers") { + return ext == ".json"; + } + return ext == ".gguf" || ext == ".safetensors" || ext == ".sft" || + ext == ".ckpt" || ext == ".pt" || ext == ".pth" || ext == ".bin"; +} + +void refresh_model_catalog(ServerRuntime& runtime) { + auto& models = *runtime.models; + std::lock_guard lock(models.catalog_mutex); + std::map recipes; + std::map files; + json errors = json::array(); + auto report = [&](const std::string& file, const std::string& message) { + errors.push_back({{"file", file}, {"message", message}}); + }; + for (const char* builtin : k_builtin_recipes) { + auto recipe = normalize_recipe(json::parse(builtin)); + const auto id = recipe.at("id").get(); + recipe["source"] = "builtin"; + recipes[id] = std::move(recipe); + } + + std::set seen_files; + for (const auto& source : get_model_roots(*runtime.svr_params)) { + const auto& root = source.path; + auto display_path = [&](const fs::path& path) { + return source.id + "/" + path.lexically_relative(root).generic_u8string(); + }; + const fs::path recipe_dir = root / "recipes"; + std::map> user_recipes; + try { + if (fs::exists(recipe_dir)) { + if (!model_path_inside(root, fs::canonical(recipe_dir))) + throw std::runtime_error("recipes directory escapes model directory"); + for (const auto& entry : fs::directory_iterator(recipe_dir)) { + const std::string display = display_path(entry.path()); + if (entry.path().extension() != ".json") + continue; + try { + if (!entry.is_regular_file() || !model_path_inside(root, fs::canonical(entry.path()))) + throw std::runtime_error("recipe must be a file inside the model directory"); + if (entry.file_size() > 1024 * 1024) + throw std::runtime_error("recipe exceeds 1 MiB"); + std::ifstream input(entry.path(), std::ios::binary); + json recipe; + input >> recipe; + recipe = normalize_recipe(std::move(recipe)); + recipe["source"] = "user"; + recipe["file"] = entry.path().filename().u8string(); + recipe["root"] = source.id; + const auto id = recipe.at("id").get(); + user_recipes[id].push_back(std::move(recipe)); + } catch (const std::exception& e) { + report(display, e.what()); + } + } + } + } catch (const std::exception& e) { + report(display_path(recipe_dir), e.what()); + } + for (auto& item : user_recipes) { + if (item.second.size() != 1) { + for (const auto& recipe : item.second) + report(source.id + "/recipes/" + recipe["file"].get(), "duplicate user recipe id: " + item.first); + continue; + } + const auto existing = recipes.find(item.first); + if (existing != recipes.end()) { + if (existing->second["source"] == "user") { + LOG_WARN("recipe '%s' from %s is shadowed by an earlier model directory", item.first.c_str(), source.label.c_str()); + continue; + } + LOG_INFO("user recipe '%s' from %s overrides the builtin", item.first.c_str(), source.label.c_str()); + } + recipes[item.first] = std::move(item.second.front()); + } + + std::set categories; + for (const auto& slot : k_model_slots) + categories.insert(slot.categories.begin(), slot.categories.end()); + for (const auto& category : categories) { + try { + for (const auto& directory : model_category_directories(source, category)) { + if (!model_path_inside(root, fs::canonical(directory))) { + report(display_path(directory), "category directory escapes model directory"); + continue; + } + for (fs::recursive_directory_iterator it(directory), end; it != end; ++it) { + std::error_code path_error; + const fs::path canonical = fs::canonical(it->path(), path_error); + if (path_error || !model_path_inside(root, canonical)) { + it.disable_recursion_pending(); + report(display_path(it->path()), path_error ? path_error.message() : "path escapes model directory"); + continue; + } + if (!it->is_regular_file() || !supported_model_file(it->path(), category) || + !seen_files.insert(model_path_key(canonical)).second) + continue; + const auto relative = it->path().lexically_relative(root).generic_u8string(); + const auto id = source.primary ? relative : source.id + "/" + relative; + files[id] = {category, it->path().lexically_relative(directory).generic_u8string(), relative, source}; + } + } + } catch (const std::exception& e) { + report(source.id + "/" + category, e.what()); + } + } + } + std::map, size_t> names; + for (const auto& entry : files) + ++names[{entry.second.category, entry.second.name}]; + for (auto& entry : files) { + auto& file = entry.second; + if (names[{file.category, file.name}] > 1) + file.name = (file.root.path / fs::u8path(file.relative_path)).generic_u8string(); + } + models.recipes = std::move(recipes); + models.files = std::move(files); + models.catalog_errors = std::move(errors); +} + +json model_status(ServerRuntime& runtime) { + std::lock_guard lock(runtime.models->status_mutex); + auto& models = *runtime.models; + json status = models.status; + if (status["state"] == "loading" && !status["load_elapsed_seconds"].is_null()) { + status["load_elapsed_seconds"] = std::chrono::duration(std::chrono::steady_clock::now() - models.load_started_at).count(); + } + return status; +} + +void initialize_server_models(ServerRuntime& runtime) { + auto& models = *runtime.models; + models.base_context = *runtime.ctx_params; + clear_model_paths(models.base_context); + models.base_defaults = *runtime.default_gen_params; + if (runtime.sd_ctx) { + models.status["state"] = "ready"; + models.status["origin"] = "command_line"; + json features = json::array(); + for (const auto& feature : k_recipe_features) + features.push_back(feature.first); + models.active_recipe = { + {"id", "@command-line"}, + {"name", "Command-line model (all features)"}, + {"source", "command_line"}, + {"description", "Uses the loaded command-line model and its generation defaults."}, + {"slots", json::array()}, + {"defaults", json::object()}, + {"features", features}, + }; + models.active_recipe["effective_features"] = resolve_recipe_features(models.active_recipe); + models.status["runtime_recipe"] = models.active_recipe; + } + refresh_model_catalog(runtime); +} + +void prepare_model_load(ServerRuntime& runtime, const json& body, SDContextParams& context, SDGenerationParams& defaults, json* loaded_recipe) { + check_keys(body, {"recipe", "models"}); + const auto recipe_id = body.at("recipe").get(); + auto selection = body.at("models"); + if (!selection.is_object()) + throw std::runtime_error("models must be an object"); + auto& models = *runtime.models; + std::lock_guard lock(models.catalog_mutex); + const auto found = models.recipes.find(recipe_id); + if (found == models.recipes.end()) + throw std::runtime_error("unknown recipe"); + const auto& recipe = found->second; + if (loaded_recipe) + *loaded_recipe = recipe; + context = models.base_context; + defaults = models.base_defaults; + apply_recipe_context(recipe.value("context", json::object()), context); + apply_recipe_defaults(recipe.value("defaults", json::object()), defaults); + std::set allowed; + for (const auto& slot : recipe.at("slots")) { + const auto id = slot.at("id").get(); + allowed.insert(id); + if (id == "tae" || id == "taesd") { + const std::string alias = id == "tae" ? "taesd" : "tae"; + allowed.insert(alias); + if (selection.contains(alias)) { + if (selection.contains(id)) + throw std::runtime_error("specify only one of tae and taesd"); + selection[id] = selection.at(alias); + } + } + const std::string file_id = selection.value(id, std::string()); + if (file_id.empty()) { + if (slot.at("required").get()) + throw std::runtime_error("missing required model: " + id); + continue; + } + const auto file = models.files.find(file_id); + if (file == models.files.end() || !matches_model_category(*find_model_slot(id), file->second.category)) { + throw std::runtime_error("unknown model or incorrect category: " + id); + } + const auto& root = file->second.root.path; + const fs::path path = fs::canonical(root / fs::u8path(file->second.relative_path)); + if (!model_path_inside(root, path) || !fs::is_regular_file(path)) { + throw std::runtime_error("model file must remain inside the model directory: " + id); + } + context.*find_model_slot(id)->field = path.u8string(); + } + for (auto it = selection.begin(); it != selection.end(); ++it) { + if (!allowed.count(it.key())) + throw std::runtime_error("unknown model slot: " + it.key()); + } + for (const auto& group : recipe.value("required_any", json::array())) { + if (std::none_of(group.begin(), group.end(), [&](const json& value) { + return !selection.value(value.get(), std::string()).empty(); + })) { + throw std::runtime_error("select at least one model from: " + group.dump()); + } + } + if (!context.resolve_and_validate(IMG_GEN)) + throw std::runtime_error("invalid model context parameters"); +} diff --git a/examples/server/model_catalog.h b/examples/server/model_catalog.h new file mode 100644 index 000000000..7d49a1924 --- /dev/null +++ b/examples/server/model_catalog.h @@ -0,0 +1,56 @@ +#ifndef __SD_EXAMPLES_SERVER_MODEL_CATALOG_H__ +#define __SD_EXAMPLES_SERVER_MODEL_CATALOG_H__ + +#include +#include +#include +#include +#include + +#include "runtime.h" + +struct ModelRoot { + std::filesystem::path path; + std::string id; + std::string label; + bool primary = false; +}; + +struct ModelFile { + std::string category; + std::string name; + std::string relative_path; + ModelRoot root; +}; + +std::string model_path_key(const std::filesystem::path& path); +bool model_path_inside(const std::filesystem::path& root, const std::filesystem::path& path); +ModelRoot make_model_root(const std::filesystem::path& path, bool primary); +std::vector get_model_roots(const SDSvrParams& params); +std::vector model_category_directories(const ModelRoot& root, const std::string& category); + +struct ServerModels { + std::shared_mutex runtime_mutex; + std::mutex catalog_mutex; + std::mutex status_mutex; + SDCtxPtr context; + SDContextParams base_context; + SDGenerationParams base_defaults; + std::map recipes; + std::map files; + std::chrono::steady_clock::time_point load_started_at; + json active_recipe = nullptr; + json catalog_errors = json::array(); + json status = {{"state", "empty"}, {"recipe", nullptr}, {"models", json::object()}, {"error", nullptr}, {"load_elapsed_seconds", nullptr}}; +}; + +void refresh_model_catalog(ServerRuntime& runtime); +json model_status(ServerRuntime& runtime); +void initialize_server_models(ServerRuntime& runtime); + +json resolve_recipe_features(const json& recipe); +json model_generation_features(const ServerRuntime& runtime); + +void prepare_model_load(ServerRuntime& runtime, const json& body, SDContextParams& context, SDGenerationParams& defaults, json* loaded_recipe = nullptr); + +#endif diff --git a/examples/server/openai_edit_policy.h b/examples/server/openai_edit_policy.h new file mode 100644 index 000000000..c963a887f --- /dev/null +++ b/examples/server/openai_edit_policy.h @@ -0,0 +1,44 @@ +#ifndef __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__ +#define __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__ + +#include "common/resource_owners.hpp" + +enum class OpenAIEditInitImageAction { + KeepExisting, + UseUploaded, + Clear, +}; + +inline bool openai_edit_uses_uploaded_image_as_init(bool supports_ref_image_conditioning, + bool has_ref_images, + bool has_mask, + bool has_explicit_strength_override) { + return !supports_ref_image_conditioning || !has_ref_images || has_mask || has_explicit_strength_override; +} + +inline OpenAIEditInitImageAction apply_openai_edit_init_image_policy( + SDImageOwner& init_image, + SDImageOwner& uploaded_image, + bool uploaded_image_was_first_reference, + bool supports_ref_image_conditioning, + bool has_ref_images, + bool has_mask, + bool has_extra_init_image_override, + bool has_explicit_strength_override) { + if (!uploaded_image_was_first_reference || has_extra_init_image_override) { + return OpenAIEditInitImageAction::KeepExisting; + } + + if (openai_edit_uses_uploaded_image_as_init(supports_ref_image_conditioning, + has_ref_images, + has_mask, + has_explicit_strength_override)) { + init_image = std::move(uploaded_image); + return OpenAIEditInitImageAction::UseUploaded; + } + + init_image.reset(); + return OpenAIEditInitImageAction::Clear; +} + +#endif // __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__ diff --git a/examples/server/recipes/README.md b/examples/server/recipes/README.md new file mode 100644 index 000000000..5043f6d1b --- /dev/null +++ b/examples/server/recipes/README.md @@ -0,0 +1,79 @@ +# Built-in recipe policy + +Read these rules before adding or changing JSON files in this directory. A recipe +represents a supported model-loading and capability configuration, not a saved +prompt, a style preset, or a collection of sampling settings. + +## Required rules + +1. **Community editing adapters get a separate recipe.** When the base model + does not natively support instruction/reference-image editing and needs a + community LoRA to gain it, provide a dedicated editing recipe. Keep editing + inputs disabled in the base recipe. Examples: `anima` / `anima-edit` and + `krea2-raw` / `krea2-raw-edit`. +2. **Split different task weights; share native multi-task weights.** If editing + and text-to-image use different generation checkpoints, use separate recipes. + If the same checkpoint natively supports both, use one recipe. Examples: + `boogu-image` / `boogu-image-edit` are separate; `flux2-dev` and + `qwen-image-21` each cover both generation and editing. Apply the same rule + to video: separate HunyuanVideo 1.5 T2V/I2V and Wan 2.2 A14B T2V/I2V weights, + but share one `wan22-ti2v-5b` recipe for its native T2V/I2V checkpoint. +3. **Capability-extending weights get a separate recipe.** ControlNet, + IP-Adapter, PuLID, PhotoMaker, motion modules, and similar adapters that add + a distinct conditioning or generation capability belong in dedicated + recipes. Make the extension's model slots required when the schema provides + them. Do not add these as optional slots on the ordinary base recipe. +4. **A LoRA alone is not a reason to split.** If adding a LoRA does not change + supported functionality, reuse the base recipe. Style, character, quality, + acceleration and distillation LoRAs do not get dedicated built-in recipes. + Users select the LoRA and change steps, guidance or other generation settings + separately. Standalone base/turbo/distilled checkpoints still follow the task, + component and layer-count rules; a different filename alone is not a reason + for another recipe. +5. **Keep native components in the model's recipe.** Native vision encoders, + projectors, connectors and other components required by the model belong in + its existing recipe. Splitting a native component into another file does not + make it a capability-extending adapter under rule 3. For example, Qwen vision + projectors and LLaDA/LTX connectors do not get separate recipes. A slot may be + optional when its weights can be bundled in another selected file. +6. **Layer-count variants share the family recipe.** Adding, repeating, removing + or pruning model layers does not justify a separate recipe when the runtime + supports the variant with the same components and capabilities. This also + applies to distilled checkpoints whose relevant difference is reduced depth. + Use `flux1-dev` for FLUX.1 Lite, `sdxl-checkpoint` for SSD-1B and Segmind Vega, + and the corresponding SD 1/2 checkpoint recipes for Tiny SD/Tiny SD2. Only an + independent difference in task, required component compatibility or latent + format can justify splitting such a variant; document that difference. + +## Interpretation and review + +- Instruction/reference-image editing is distinct from generic image-to-image + denoising. The shared `img2img` capability does not imply native editing. +- Native components and external capability adapters are distinguished by their + role, not by whether their weights are stored in a separate file. LLaDA's + SigVQ-containing connectors remain in its shared generation/editing recipe. +- Do not merge models solely because they descend from the same family. Flex.2 + has different native inpainting/control conditioning, beyond its layer count + ([model card](https://huggingface.co/ostris/Flex.2-preview)). SDXS uses an + adjusted AutoEncoderTiny and a one-step model, rather than only a shallower + ordinary SD checkpoint ([runtime guide](../../../docs/distilled_sd.md#sdxs-512-dreamshaper)). + Their native components stay in their own model recipe. +- Quantization, file format, prompt wording, seeds, step counts and CFG values + are not grounds for another recipe. Differences in required components, + latent formats or supported capabilities must be documented. +- Evaluate what an adapter actually changes. Do not infer new functionality just + from a LoRA's name or from the presence of another weight file. In particular, + do not create an Omni-Kontext recipe solely because a LoRA is selected. +- A recipe does not load generation LoRAs automatically. Community-edit recipes + must explain that the matching LoRA is required and must be selected separately. +- Only advertise features implemented by the current runtime. Check the relevant + model documentation and code, and match each task's slots and feature flags to + its weights. Record any unverified inference paths in the change description. +- Before submitting, validate all recipe JSON, inspect the embedded recipe API, + and check that task-specific inputs appear only on the appropriate recipes. + A catalog check is not a substitute for model-generation verification. + +These are built-in catalog rules. Users may keep personal presets in +`/recipes/`; they need not add those presets to the built-in list. +See [the recipe schema and feature reference](../api.md#user-recipe-format) +for supported fields. diff --git a/examples/server/recipes/anima-edit.json b/examples/server/recipes/anima-edit.json new file mode 100644 index 000000000..f22c1b6cc --- /dev/null +++ b/examples/server/recipes/anima-edit.json @@ -0,0 +1,44 @@ +{ + "schema_version": 1, + "id": "anima-edit", + "name": "Anima Edit (LoRA)", + "description": "Requires a matching community editing LoRA. Apply it after loading and supply reference images; uses the cosmos_reference preset.", + "model_url": "https://huggingface.co/circlestone-labs/Anima", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 0.6B Base Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "scheduler": "discrete", + "cfg_scale": 4, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "flow_shift", + "ref_images" + ] +} diff --git a/examples/server/recipes/anima.json b/examples/server/recipes/anima.json new file mode 100644 index 000000000..4809bd4bf --- /dev/null +++ b/examples/server/recipes/anima.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "id": "anima", + "name": "Anima", + "description": "Supports all Anima variants supported by sd.cpp. Use a compatible Qwen Image VAE or TAE; adjust generation settings for distilled variants.", + "model_url": "https://huggingface.co/circlestone-labs/Anima", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 0.6B Base Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 4, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes/animatediff-sd15.json b/examples/server/recipes/animatediff-sd15.json new file mode 100644 index 000000000..a7c8338f1 --- /dev/null +++ b/examples/server/recipes/animatediff-sd15.json @@ -0,0 +1,44 @@ +{ + "schema_version": 1, + "id": "animatediff-sd15", + "name": "AnimateDiff SD 1.5", + "description": "Use Video Generation with an SD 1.5 checkpoint and compatible motion module.", + "model_url": "https://huggingface.co/guoyww/animatediff", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "motion_module", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "sampling_method": "euler", + "video_frames": 8, + "fps": 8, + "cfg_scale": 7.5, + "steps": 20, + "width": 512, + "height": 512 + }, + "features": [ + "cache_unet", + "clip_skip", + "init_image" + ], + "disabled_features": [ + "cache_dit", + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/boogu-image-edit.json b/examples/server/recipes/boogu-image-edit.json new file mode 100644 index 000000000..ff1aaeefb --- /dev/null +++ b/examples/server/recipes/boogu-image-edit.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "id": "boogu-image-edit", + "name": "Boogu Image Edit", + "description": "Supply reference images. The separate vision projector is needed only when it is not bundled in the text encoder.", + "model_url": "https://huggingface.co/Boogu/Boogu-Image-0.1-Edit", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 8B Text Encoder" + }, + { + "id": "llm_vision", + "required": false, + "label": "Vision projector (required for separate GGUF text encoders)" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 5, + "steps": 20, + "width": 1024, + "height": 1024, + "img_cfg_scale": 1, + "sampling_method": "euler" + }, + "features": [ + "flow_shift", + "img_cfg", + "ref_images" + ] +} diff --git a/examples/server/recipes/boogu-image.json b/examples/server/recipes/boogu-image.json new file mode 100644 index 000000000..24c6a42b2 --- /dev/null +++ b/examples/server/recipes/boogu-image.json @@ -0,0 +1,41 @@ +{ + "schema_version": 1, + "id": "boogu-image", + "name": "Boogu Image Base", + "model_url": "https://huggingface.co/Boogu/Boogu-Image-0.1-Base", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 8B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 4, + "steps": 20, + "width": 1024, + "height": 1024, + "sampling_method": "euler" + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes/chroma-radiance.json b/examples/server/recipes/chroma-radiance.json new file mode 100644 index 000000000..6a0be8a56 --- /dev/null +++ b/examples/server/recipes/chroma-radiance.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "id": "chroma-radiance", + "name": "Chroma Radiance", + "description": "Pixel-space generation without a VAE. Also accepts the x0 and experimental patch-32 variants.", + "model_url": "https://huggingface.co/lodestones/Chroma1-Radiance", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "t5xxl", + "required": true + } + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 3.5, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "flow_shift", + "slg" + ], + "disabled_features": [ + "preview", + "vae_tiling" + ] +} diff --git a/examples/server/recipes/chroma.json b/examples/server/recipes/chroma.json new file mode 100644 index 000000000..b8d1da080 --- /dev/null +++ b/examples/server/recipes/chroma.json @@ -0,0 +1,41 @@ +{ + "schema_version": 1, + "id": "chroma", + "name": "Chroma", + "model_url": "https://huggingface.co/lodestones/Chroma1-HD", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "t5xxl", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 3, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "flow_shift", + "slg" + ] +} diff --git a/examples/server/recipes/cosxl-edit.json b/examples/server/recipes/cosxl-edit.json new file mode 100644 index 000000000..49407fccd --- /dev/null +++ b/examples/server/recipes/cosxl-edit.json @@ -0,0 +1,36 @@ +{ + "schema_version": 1, + "id": "cosxl-edit", + "name": "CosXL Edit", + "description": "Supply the image to edit as a reference image.", + "model_url": "https://huggingface.co/stabilityai/cosxl", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "img_cfg_scale": 1.5, + "sampling_method": "euler_a", + "scheduler": "karras" + }, + "features": [ + "cache_unet", + "clip_skip", + "img_cfg", + "ref_images", + "shifted_timestep" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/defaults.md b/examples/server/recipes/defaults.md new file mode 100644 index 000000000..2be4dfbf6 --- /dev/null +++ b/examples/server/recipes/defaults.md @@ -0,0 +1,124 @@ +# Built-in generation defaults + +Reviewed against model-publisher cards and reference implementations on 2026-10-20. + +These are starting values, not limits on supported resolutions or generation settings. +Built-in step defaults above 20 are reduced to 20; the cited sources may recommend more steps. +Users can still choose a higher step count. +When several aspect ratios or ranges are published, the selected example or square bucket is noted below. +A dash means no recipe override: the available primary sources did not establish a fixed value, or the size depends on the input image. +Ordinary DiT recipes explicitly use Euler; PiD retains its LCM sampling path. +Guidance-distilled models use `cfg_scale: 1` to disable external CFG in sd.cpp. FLUX embedded guidance is stored separately as `distilled_guidance`. +For community edit adapters and other optional extensions, these are the base-model settings unless the source specifies otherwise. + +| Recipe | CFG | Steps | Width Γ— height | Source / notes | +| --- | --- | --- | --- | --- | +| `anima` | 4 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/circlestone-labs/Anima) The official ranges are 30–50 steps / CFG 4–5; CFG 4 selects the lower end. Edit uses the same base; tune the chosen editing LoRA separately. | +| `anima-edit` | 4 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/circlestone-labs/Anima) The official ranges are 30–50 steps / CFG 4–5; CFG 4 selects the lower end. Edit uses the same base; tune the chosen editing LoRA separately. | +| `animatediff-sd15` | 7.5 | 20 | 512 Γ— 512 | [Source](https://raw.githubusercontent.com/guoyww/AnimateDiff/main/configs/prompts/1_animate/1_3_animate_ToonYou.yaml) | +| `boogu-image` | 4 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/Boogu/Boogu-Image-0.1-Base) | +| `boogu-image-edit` | 5 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/Boogu/Boogu-Image-0.1-Edit) | +| `chroma` | 3 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/lodestones/Chroma1-HD/raw/main/README.md) | +| `chroma-radiance` | 3.5 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/lodestones/Chroma1-Radiance/raw/main/inference_reference_code.py) | +| `cosxl-edit` | β€” | β€” | β€” | [Source](https://huggingface.co/stabilityai/cosxl) No numerical generation defaults in the author’s model card; previous test values were removed. | +| `ernie-image` | 4 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/baidu/ERNIE-Image) | +| `ernie-image-turbo` | 1 | 8 | 1024 Γ— 1024 | [Source](https://huggingface.co/baidu/ERNIE-Image-Turbo) | +| `flex2` | 1 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/ostris/Flex.2-preview) | +| `flux1-canny` | 1 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-Canny-dev) | +| `flux1-depth` | 1 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-Depth-dev) | +| `flux1-dev` | 1 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-dev) | +| `flux1-fill` | 1 | 20 | 1232 Γ— 1632 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-Fill-dev) | +| `flux1-kontext` | 1 | 20 | β€” | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/flux/pipeline_flux_kontext.py) Output size normally follows the reference image; no fixed size is imposed by this recipe. | +| `flux1-pulid` | 1 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-dev) FLUX.1-dev defaults; identity conditioning is provided by PuLID. | +| `flux1-schnell` | 1 | 4 | 1024 Γ— 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-schnell) | +| `flux2-dev` | 1 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.2-dev) | +| `flux2-klein-4b` | 1 | 4 | 1024 Γ— 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.2-klein-4B) | +| `flux2-klein-9b` | 1 | 4 | 1024 Γ— 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.2-klein-9B) | +| `flux2-klein-base-4b` | 4 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.2-klein-base-4B) | +| `flux2-klein-base-9b` | 4 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.2-klein-base-9B) | +| `flux3-klein` | 1 | 16 | 1024 Γ— 1024 | [Source](https://raw.githubusercontent.com/black-forest-labs/flux-3-image-klein/main/README.md) | +| `flux3-klein-base` | 4 | 20 | 1024 Γ— 1024 | [Source](https://raw.githubusercontent.com/black-forest-labs/flux-3-image-klein/main/src/flux3/pipeline.py) | +| `hidream-o1-dev` | 1 | 20 | 2048 Γ— 2048 | [Source](https://huggingface.co/HiDream-ai/HiDream-O1-Image-Dev) | +| `hunyuan-video-15-i2v` | 6 | 20 | 1280 Γ— 720 | [Source](https://huggingface.co/tencent/HunyuanVideo-1.5) 720p base I2V defaults; CFG/step-distilled and 480p checkpoints use different settings. | +| `hunyuan-video-15-t2v` | 6 | 20 | 1280 Γ— 720 | [Source](https://huggingface.co/tencent/HunyuanVideo-1.5) 720p base T2V defaults; CFG-distilled / 480p checkpoints need their published settings. | +| `ideogram4` | 7 | 20 | 1024 Γ— 1024 | [Source](https://raw.githubusercontent.com/ideogram-oss/ideogram4/main/docs/inference.md) Constant-guidance pipeline defaults; this recipe does not reproduce the CLI’s per-step guidance schedule. | +| `instruct-pix2pix` | β€” | 10 | β€” | [Source](https://huggingface.co/timbrooks/instruct-pix2pix) Official quick-start example; output size follows the input image. | +| `iris-3b` | 3 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/speridlabs/iris-3b) | +| `krea2-raw` | 3.5 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/krea/Krea-2-Raw) Base model defaults; edit requires a matching community LoRA with its own settings. | +| `krea2-raw-edit` | 3.5 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/krea/Krea-2-Raw) Base model defaults; edit requires a matching community LoRA with its own settings. | +| `krea2-turbo` | 1 | 8 | 2048 Γ— 2048 | [Source](https://huggingface.co/krea/Krea-2-Turbo) Official CLI example; edit requires a matching community LoRA. | +| `krea2-turbo-edit` | 1 | 8 | 2048 Γ— 2048 | [Source](https://huggingface.co/krea/Krea-2-Turbo) Official CLI example; edit requires a matching community LoRA. | +| `lens` | 5 | 20 | 1440 Γ— 1440 | [Source](https://huggingface.co/microsoft/Lens) Official model card: RL model, default base_resolution 1440. | +| `lens-turbo` | 1 | 4 | 1440 Γ— 1440 | [Source](https://huggingface.co/microsoft/Lens-Turbo) Official model card: 4-step distilled model, default base_resolution 1440. | +| `lingbot-video-dense` | 3 | 20 | 832 Γ— 480 | [Source](https://huggingface.co/robbyant/lingbot-video-dense-1.3b) Official first-stage inference example; optional refiner is separate. | +| `lingbot-video-moe` | 3 | 20 | 832 Γ— 480 | [Source](https://huggingface.co/robbyant/lingbot-video-moe-30b-a3b) Official inference example. | +| `llada-image` | 5 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/inclusionAI/LLaDA-Image) | +| `llada-image-turbo` | 1 | 4 | 1024 Γ— 1024 | [Source](https://huggingface.co/inclusionAI/LLaDA-Image-Turbo) | +| `longcat-image` | 4 | 20 | 1344 Γ— 768 | [Source](https://huggingface.co/meituan-longcat/LongCat-Image) | +| `longcat-image-edit` | 4.5 | 20 | β€” | [Source](https://huggingface.co/meituan-longcat/LongCat-Image-Edit) Reference-image-dependent output size; no fixed size is imposed. | +| `ltx23-dev` | 3 | 20 | 768 Γ— 512 | [Source](https://raw.githubusercontent.com/Lightricks/LTX-2/main/packages/ltx-pipelines/src/ltx_pipelines/utils/constants.py) Official single-stage parameters, before optional latent upscaling. | +| `ltx23-distilled` | 1 | 8 | 768 Γ— 512 | [Source](https://raw.githubusercontent.com/Lightricks/LTX-2/main/packages/ltx-pipelines/src/ltx_pipelines/utils/constants.py) Model card specifies 8 steps / CFG 1; size uses official pipeline’s first stage. | +| `ltx25-dev` | 3 | 20 | 768 Γ— 512 | [Source](https://raw.githubusercontent.com/Lightricks/LTX-2/main/packages/ltx-pipelines/src/ltx_pipelines/utils/constants.py) Official non-distilled pipeline parameters for model versions >= 2.4, before optional upscaling. | +| `mage-flow` | 5 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/microsoft/Mage-Flow) | +| `mage-flow-edit` | 5 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/microsoft/Mage-Flow) | +| `mage-flow-turbo` | 1 | 4 | 1024 Γ— 1024 | [Source](https://huggingface.co/microsoft/Mage-Flow) | +| `ming-image` | 1 | 12 | 2048 Γ— 2048 | [Source](https://huggingface.co/inclusionAI/Ming-Image-0.1-Design) | +| `minimax-h3-fl2va` | 1 | β€” | 1344 Γ— 768 | [Source](https://huggingface.co/docs/diffusers/main/en/api/pipelines/minimax_h3) Guidance-distilled; official 768p canvas. No fixed step count is prescribed here; the upstream API counts sigma grid points rather than sd.cpp denoising steps. | +| `minimax-h3-ref2va` | 1 | β€” | 1344 Γ— 768 | [Source](https://huggingface.co/docs/diffusers/main/en/api/pipelines/minimax_h3) Guidance-distilled; official 768p canvas. No fixed step count is prescribed here; the upstream API counts sigma grid points rather than sd.cpp denoising steps. | +| `minit2i` | 2.5 | 20 | 512 Γ— 512 | [Source](https://huggingface.co/MiniT2I/MiniT2I/raw/main/pipeline.py) B/16 example and native model size; L/16 uses CFG 6. | +| `moss-image-base` | 3 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base) | +| `moss-image-sft` | 3 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/OpenMOSS-Team/MOSS-Image-SFT) | +| `nitrosd-realism` | 1 | 1 | 1024 Γ— 1024 | [Source](https://huggingface.co/ChenDY/NitroFusion) | +| `ovis-image` | 5 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/ATH-MaaS/Ovis-Image-7B) | +| `pid` | 1 | 4 | 2048 Γ— 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. | +| `pid-15` | 1 | 4 | 2048 Γ— 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. | +| `pid-15-flux2` | 1 | 4 | 2048 Γ— 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. | +| `pid-15-qwen-image` | 1 | 4 | 2048 Γ— 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. | +| `pid-15-sd3` | 1 | 4 | 2048 Γ— 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. | +| `pid-flux2` | 1 | 4 | 2048 Γ— 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. | +| `pid-qwen-image` | 1 | 4 | 2048 Γ— 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. | +| `pid-sd3` | 1 | 4 | 2048 Γ— 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. | +| `pixart-alpha-1024` | 4.5 | 20 | 1024 Γ— 1024 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py) | +| `pixart-alpha-512` | 4.5 | 20 | 512 Γ— 512 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py) | +| `pixart-sigma` | 4.5 | 20 | 1024 Γ— 1024 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_sigma.py) | +| `qwen-image` | 4 | 20 | 1328 Γ— 1328 | [Source](https://huggingface.co/Qwen/Qwen-Image) Official square resolution bucket; true_cfg_scale maps to cfg_scale. | +| `qwen-image-21` | 1 | 20 | 2048 Γ— 2048 | [Source](https://raw.githubusercontent.com/QwenLM/Qwen-Image-2.1/main/README.md) Original checkpoint defaults; select the matching TAEQI2.1 to replace its VAE. | +| `qwen-image-21-turbo` | 1 | 8 | 2048 Γ— 2048 | [Model card](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo), [fixed sigmas](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo/blob/main/model_index.json). Standalone Turbo weights; the fixed 8-step schedule is applied on model load. | +| `qwen-image-edit` | 4 | 20 | β€” | [Source](https://huggingface.co/Qwen/Qwen-Image-Edit) Official editing example; output dimensions follow reference-image aspect ratio. | +| `qwen-image-edit-2509` | 4 | 20 | β€” | [Source](https://huggingface.co/Qwen/Qwen-Image-Edit-2509) Official editing example; output dimensions follow reference-image aspect ratio. | +| `qwen-image-edit-2511` | 4 | 20 | β€” | [Source](https://huggingface.co/Qwen/Qwen-Image-Edit-2511) Official editing example; output dimensions follow reference-image aspect ratio. | +| `qwen-image-layered` | 4 | 20 | 640 Γ— 640 | [Source](https://huggingface.co/Qwen/Qwen-Image-Layered) Official recommended 640 resolution bucket; layered RGBA VAE is required. | +| `sd-turbo` | 1 | 1 | 512 Γ— 512 | [Source](https://huggingface.co/stabilityai/sd-turbo) | +| `sd1-checkpoint` | 7.5 | 20 | 512 Γ— 512 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py) | +| `sd1-controlnet` | 7.5 | 20 | 512 Γ— 512 | [Source](https://huggingface.co/lllyasviel/control_v11p_sd15_canny) | +| `sd1-inpainting` | 7.5 | 20 | 512 Γ— 512 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py) | +| `sd1-ip-adapter` | 7.5 | 20 | 512 Γ— 512 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py) SD 1.x base pipeline defaults; adapter strength is configured separately. | +| `sd2-512-checkpoint` | 7.5 | 20 | 512 Γ— 512 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py) | +| `sd2-768-checkpoint` | 7.5 | 20 | 768 Γ— 768 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py) | +| `sd2-inpainting` | 7.5 | 20 | 512 Γ— 512 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py) | +| `sd3-medium` | 7 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/stabilityai/stable-diffusion-3-medium-diffusers) | +| `sd35-large` | 3.5 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/stabilityai/stable-diffusion-3.5-large) | +| `sd35-medium` | 4.5 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/stabilityai/stable-diffusion-3.5-medium) | +| `sdxl-checkpoint` | 5 | 20 | 1024 Γ— 1024 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py) | +| `sdxl-controlnet` | 5 | 20 | 1024 Γ— 1024 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py) SDXL pipeline defaults; compatible ControlNet weights are required. | +| `sdxl-inpainting` | 8 | 20 | 1024 Γ— 1024 | [Source](https://huggingface.co/diffusers/stable-diffusion-xl-1.0-inpainting-0.1/raw/main/README.md) | +| `sdxl-ip-adapter` | 5 | 20 | 1024 Γ— 1024 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py) SDXL base pipeline defaults; adapter strength is configured separately. | +| `sdxl-photomaker` | β€” | β€” | 1024 Γ— 1024 | [Source](https://huggingface.co/TencentARC/PhotoMaker) Use settings for the selected compatible SDXL checkpoint; no unique CFG/steps are prescribed by the model card. | +| `sdxl-turbo` | 1 | 1 | 512 Γ— 512 | [Source](https://huggingface.co/stabilityai/sdxl-turbo) | +| `sdxs` | 1 | 1 | 512 Γ— 512 | [Source](https://huggingface.co/IDKiro/sdxs-512-dreamshaper) | +| `sefi-image` | 4 | 20 | β€” | [Source](https://huggingface.co/SeFi-Image/SeFi-Image-5B-Base) Base/RL family defaults; Turbo checkpoints require 4 steps / CFG 1. | +| `sensenova-u15` | 4 | 20 | 2048 Γ— 2048 | [Source](https://raw.githubusercontent.com/OpenSenseNova/SenseNova-U1/main/examples/README.md) | +| `wan21-flf2v` | 5 | 20 | 1280 Γ— 720 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.1/main/generate.py) | +| `wan21-i2v` | 5 | 20 | 832 Γ— 480 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.1/main/generate.py) 480p base defaults; also accepts compatible SkyReels V2 and single Wan 2.2 I2V experts. | +| `wan21-t2v` | 5 | 20 | 1280 Γ— 720 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.1/main/generate.py) 14B base defaults; single Wan 2.2 experts and acceleration LoRAs need matching settings. | +| `wan21-vace` | 5 | 20 | 832 Γ— 480 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.1/main/generate.py) | +| `wan22-i2v` | 3.5 | 20 | 1280 Γ— 720 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/wan/configs/wan_i2v_A14B.py) | +| `wan22-s2v` | 4.5 | 20 | β€” | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/wan/configs/wan_s2v_14B.py) The official S2V pipeline derives output dimensions from the input. | +| `wan22-t2v` | 3 | 20 | 1280 Γ— 720 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/wan/configs/wan_t2v_A14B.py) | +| `wan22-ti2v-5b` | 5 | 20 | 1280 Γ— 704 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/wan/configs/wan_ti2v_5B.py) | +| `z-image` | 4 | 20 | 720 Γ— 1280 | [Source](https://huggingface.co/Tongyi-MAI/Z-Image) | +| `z-image-l2p` | β€” | β€” | β€” | [Source](https://huggingface.co/zhen-nan/L2P) No numerical inference recommendations in the author’s model card; previous test values were removed. | +| `z-image-turbo` | 1 | 9 | 1024 Γ— 1024 | [Source](https://huggingface.co/Tongyi-MAI/Z-Image-Turbo) | + +TAE alternatives use the model-compatible weights listed by [TAESD](https://github.com/madebyollin/taesd) and [TAEHV](https://github.com/madebyollin/taehv). +TAE slots are omitted for pixel-space models, PiD, layered RGBA decoding and Mage-VAE paths that the current runtime does not map to a compatible TAE. diff --git a/examples/server/recipes/ernie-image-turbo.json b/examples/server/recipes/ernie-image-turbo.json new file mode 100644 index 000000000..5702f5efc --- /dev/null +++ b/examples/server/recipes/ernie-image-turbo.json @@ -0,0 +1,41 @@ +{ + "schema_version": 1, + "id": "ernie-image-turbo", + "name": "ERNIE Image Turbo", + "model_url": "https://huggingface.co/baidu/ERNIE-Image-Turbo", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Ministral 3 3B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 1, + "steps": 8, + "width": 1024, + "height": 1024, + "sampling_method": "euler" + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes/ernie-image.json b/examples/server/recipes/ernie-image.json new file mode 100644 index 000000000..a61000008 --- /dev/null +++ b/examples/server/recipes/ernie-image.json @@ -0,0 +1,41 @@ +{ + "schema_version": 1, + "id": "ernie-image", + "name": "ERNIE Image", + "model_url": "https://huggingface.co/baidu/ERNIE-Image", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Ministral 3 3B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 4, + "steps": 20, + "width": 1024, + "height": 1024, + "sampling_method": "euler" + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes/flex2.json b/examples/server/recipes/flex2.json new file mode 100644 index 000000000..bbd900c39 --- /dev/null +++ b/examples/server/recipes/flex2.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "id": "flex2", + "name": "Flex.2 Preview", + "model_url": "https://huggingface.co/ostris/Flex.2-preview", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "clip_l", + "required": true + }, + { + "id": "t5xxl", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 1, + "steps": 20, + "width": 1024, + "height": 1024, + "distilled_guidance": 3.5 + }, + "features": [ + "clip_skip", + "distilled_guidance", + "flow_shift", + "slg" + ] +} diff --git a/examples/server/recipes/flux1-canny.json b/examples/server/recipes/flux1-canny.json new file mode 100644 index 000000000..de9bb609a --- /dev/null +++ b/examples/server/recipes/flux1-canny.json @@ -0,0 +1,53 @@ +{ + "schema_version": 1, + "id": "flux1-canny", + "name": "FLUX.1 Canny dev", + "description": "Select Canny weights and supply a precomputed edge map as Control Image. Edge detection is not performed by this recipe.", + "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-Canny-dev", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "clip_l", + "required": true + }, + { + "id": "t5xxl", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 1, + "steps": 20, + "width": 1024, + "height": 1024, + "distilled_guidance": 30 + }, + "features": [ + "clip_skip", + "distilled_guidance", + "flow_shift", + "slg", + "control_image" + ], + "disabled_features": [ + "mask_image" + ] +} diff --git a/examples/server/recipes/flux1-depth.json b/examples/server/recipes/flux1-depth.json new file mode 100644 index 000000000..0efefd275 --- /dev/null +++ b/examples/server/recipes/flux1-depth.json @@ -0,0 +1,54 @@ +{ + "schema_version": 1, + "id": "flux1-depth", + "name": "FLUX.1 Depth dev", + "description": "Select Depth weights and supply a precomputed depth map as Control Image. Depth estimation is not performed by this recipe.", + "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-Depth-dev", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "clip_l", + "required": true + }, + { + "id": "t5xxl", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "distilled_guidance": 10, + "scheduler": "flux", + "cfg_scale": 1, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "clip_skip", + "distilled_guidance", + "flow_shift", + "slg", + "control_image" + ], + "disabled_features": [ + "mask_image" + ] +} diff --git a/examples/server/recipes/flux1-dev.json b/examples/server/recipes/flux1-dev.json new file mode 100644 index 000000000..93561958a --- /dev/null +++ b/examples/server/recipes/flux1-dev.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "id": "flux1-dev", + "name": "FLUX.1 dev", + "description": "Supports FLUX.1 dev and compatible layer-count variants such as FLUX.1 Lite. Select the transformer with matching CLIP-L, T5XXL and FLUX.1 VAE.", + "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-dev", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "clip_l", + "required": true + }, + { + "id": "t5xxl", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 1, + "steps": 20, + "width": 1024, + "height": 1024, + "distilled_guidance": 3.5 + }, + "features": [ + "clip_skip", + "distilled_guidance", + "flow_shift", + "slg" + ] +} diff --git a/examples/server/recipes/flux1-fill.json b/examples/server/recipes/flux1-fill.json new file mode 100644 index 000000000..5e6485ad1 --- /dev/null +++ b/examples/server/recipes/flux1-fill.json @@ -0,0 +1,52 @@ +{ + "schema_version": 1, + "id": "flux1-fill", + "name": "FLUX.1 Fill dev", + "description": "Select Fill weights and supply Init Image plus Mask Image for inpainting. Denoising strength defaults to 1 for a full denoising schedule.", + "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-Fill-dev", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "clip_l", + "required": true + }, + { + "id": "t5xxl", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "distilled_guidance": 30, + "strength": 1.0, + "scheduler": "flux", + "cfg_scale": 1, + "steps": 20, + "width": 1232, + "height": 1632 + }, + "features": [ + "clip_skip", + "distilled_guidance", + "flow_shift", + "slg", + "img_cfg" + ] +} diff --git a/examples/server/recipes/flux1-kontext.json b/examples/server/recipes/flux1-kontext.json new file mode 100644 index 000000000..d66e41a26 --- /dev/null +++ b/examples/server/recipes/flux1-kontext.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "id": "flux1-kontext", + "name": "FLUX.1 Kontext dev", + "description": "Select FLUX.1 Kontext weights. Supports text-to-image and native reference-image editing; an editing LoRA is not required.", + "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-Kontext-dev", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "clip_l", + "required": true + }, + { + "id": "t5xxl", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 1, + "steps": 20, + "distilled_guidance": 2.5 + }, + "features": [ + "clip_skip", + "distilled_guidance", + "flow_shift", + "img_cfg", + "ref_images", + "slg" + ] +} diff --git a/examples/server/recipes/flux1-pulid.json b/examples/server/recipes/flux1-pulid.json new file mode 100644 index 000000000..8203dd1b5 --- /dev/null +++ b/examples/server/recipes/flux1-pulid.json @@ -0,0 +1,56 @@ +{ + "schema_version": 1, + "id": "flux1-pulid", + "name": "FLUX.1 dev PuLID", + "description": "Supply a precomputed PuLID identity embedding with the generation request.", + "model_url": "https://huggingface.co/guozinan/PuLID", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "clip_l", + "required": true + }, + { + "id": "t5xxl", + "required": true + }, + { + "id": "pulid_weights", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 1, + "steps": 20, + "width": 1024, + "height": 1024, + "distilled_guidance": 3.5 + }, + "features": [ + "clip_skip", + "distilled_guidance", + "flow_shift", + "slg" + ], + "disabled_features": [ + "slg" + ] +} diff --git a/examples/server/recipes/flux1-schnell.json b/examples/server/recipes/flux1-schnell.json new file mode 100644 index 000000000..106fef550 --- /dev/null +++ b/examples/server/recipes/flux1-schnell.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "id": "flux1-schnell", + "name": "FLUX.1 schnell", + "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-schnell", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "clip_l", + "required": true + }, + { + "id": "t5xxl", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 1, + "steps": 4, + "width": 1024, + "height": 1024 + }, + "features": [ + "clip_skip", + "flow_shift", + "slg" + ] +} diff --git a/examples/server/recipes/flux2-dev.json b/examples/server/recipes/flux2-dev.json new file mode 100644 index 000000000..4d27ea16b --- /dev/null +++ b/examples/server/recipes/flux2-dev.json @@ -0,0 +1,47 @@ +{ + "schema_version": 1, + "id": "flux2-dev", + "name": "FLUX.2 dev", + "description": "Supports text-to-image and reference-image editing.", + "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-dev", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Mistral Small 3.2 24B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "distilled_guidance": 4, + "sampling_method": "euler", + "cfg_scale": 1, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "distilled_guidance", + "flow_shift", + "img_cfg", + "ref_images", + "slg" + ] +} diff --git a/examples/server/recipes/flux2-klein-4b.json b/examples/server/recipes/flux2-klein-4b.json new file mode 100644 index 000000000..f09360cfb --- /dev/null +++ b/examples/server/recipes/flux2-klein-4b.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "id": "flux2-klein-4b", + "name": "FLUX.2 Klein 4B", + "description": "Supports reference-image editing. The FLUX.2 small decoder is an alternative VAE.", + "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-4B", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 4B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 1, + "steps": 4, + "width": 1024, + "height": 1024, + "sampling_method": "euler" + }, + "features": [ + "flow_shift", + "img_cfg", + "ref_images", + "slg" + ] +} diff --git a/examples/server/recipes/flux2-klein-9b.json b/examples/server/recipes/flux2-klein-9b.json new file mode 100644 index 000000000..ad10d6e51 --- /dev/null +++ b/examples/server/recipes/flux2-klein-9b.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "id": "flux2-klein-9b", + "name": "FLUX.2 Klein 9B", + "description": "Supports text-to-image and reference-image editing.", + "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-9B", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 8B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 1, + "steps": 4, + "width": 1024, + "height": 1024, + "sampling_method": "euler" + }, + "features": [ + "flow_shift", + "img_cfg", + "ref_images", + "slg" + ] +} diff --git a/examples/server/recipes/flux2-klein-base-4b.json b/examples/server/recipes/flux2-klein-base-4b.json new file mode 100644 index 000000000..56c69f980 --- /dev/null +++ b/examples/server/recipes/flux2-klein-base-4b.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "id": "flux2-klein-base-4b", + "name": "FLUX.2 Klein Base 4B", + "description": "Supports text-to-image and reference-image editing.", + "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-base-4B", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 4B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 4, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "flow_shift", + "img_cfg", + "ref_images", + "slg" + ] +} diff --git a/examples/server/recipes/flux2-klein-base-9b.json b/examples/server/recipes/flux2-klein-base-9b.json new file mode 100644 index 000000000..e9bd53c6c --- /dev/null +++ b/examples/server/recipes/flux2-klein-base-9b.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "id": "flux2-klein-base-9b", + "name": "FLUX.2 Klein Base 9B", + "description": "Use the 9B base transformer with Qwen 3 8B. Supports reference-image editing.", + "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-base-9B", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 8B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 4, + "steps": 20, + "width": 1024, + "height": 1024, + "sampling_method": "euler" + }, + "features": [ + "flow_shift", + "img_cfg", + "ref_images", + "slg" + ] +} diff --git a/examples/server/recipes/flux3-klein-base.json b/examples/server/recipes/flux3-klein-base.json new file mode 100644 index 000000000..29f6f6819 --- /dev/null +++ b/examples/server/recipes/flux3-klein-base.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "id": "flux3-klein-base", + "name": "FLUX.3 Image Klein Base", + "model_url": "https://huggingface.co/black-forest-labs/flux-3-image-klein-base", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 4B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 4, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "flow_shift", + "ref_images" + ] +} diff --git a/examples/server/recipes/flux3-klein.json b/examples/server/recipes/flux3-klein.json new file mode 100644 index 000000000..164bce12a --- /dev/null +++ b/examples/server/recipes/flux3-klein.json @@ -0,0 +1,43 @@ +{ + "schema_version": 1, + "id": "flux3-klein", + "name": "FLUX.3 Image Klein", + "description": "Supports text-to-image and reference-image editing.", + "model_url": "https://huggingface.co/black-forest-labs/flux-3-image-klein", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 4B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 1, + "steps": 16, + "width": 1024, + "height": 1024 + }, + "features": [ + "flow_shift", + "ref_images" + ] +} diff --git a/examples/server/recipes/hidream-o1-dev.json b/examples/server/recipes/hidream-o1-dev.json new file mode 100644 index 000000000..8e95f5cea --- /dev/null +++ b/examples/server/recipes/hidream-o1-dev.json @@ -0,0 +1,24 @@ +{ + "schema_version": 1, + "id": "hidream-o1-dev", + "name": "HiDream O1 Image dev", + "description": "Bundled checkpoint; also supports reference-image editing.", + "model_url": "https://huggingface.co/HiDream-ai/HiDream-O1-Image-Dev", + "slots": [ + { + "id": "model", + "required": true + } + ], + "defaults": { + "cfg_scale": 1, + "steps": 20, + "width": 2048, + "height": 2048, + "sampling_method": "euler" + }, + "features": [ + "flow_shift", + "ref_images" + ] +} diff --git a/examples/server/recipes/hunyuan-video-15-i2v.json b/examples/server/recipes/hunyuan-video-15-i2v.json new file mode 100644 index 000000000..a4d672f7b --- /dev/null +++ b/examples/server/recipes/hunyuan-video-15-i2v.json @@ -0,0 +1,56 @@ +{ + "schema_version": 1, + "id": "hunyuan-video-15-i2v", + "name": "HunyuanVideo 1.5 I2V", + "description": "Select HunyuanVideo 1.5 I2V weights and supply a start image in Video Generation.", + "model_url": "https://huggingface.co/tencent/HunyuanVideo-1.5", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 2.5 VL 7B Text Encoder" + }, + { + "id": "t5xxl", + "required": true, + "label": "ByT5 Small GlyphXL" + }, + { + "id": "vae", + "required": false, + "label": "HunyuanVideo 1.5 VAE" + }, + { + "id": "tae", + "required": false, + "label": "TAEHV 1.5" + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "video_frames": 33, + "cfg_scale": 6, + "steps": 20, + "width": 1280, + "height": 720, + "flow_shift": 7 + }, + "features": [ + "flow_shift", + "init_image" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/hunyuan-video-15-t2v.json b/examples/server/recipes/hunyuan-video-15-t2v.json new file mode 100644 index 000000000..d477b8063 --- /dev/null +++ b/examples/server/recipes/hunyuan-video-15-t2v.json @@ -0,0 +1,55 @@ +{ + "schema_version": 1, + "id": "hunyuan-video-15-t2v", + "name": "HunyuanVideo 1.5 T2V", + "description": "Select HunyuanVideo 1.5 T2V weights and use Video Generation. Use the separate I2V recipe for I2V weights.", + "model_url": "https://huggingface.co/tencent/HunyuanVideo-1.5", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 2.5 VL 7B Text Encoder" + }, + { + "id": "t5xxl", + "required": true, + "label": "ByT5 Small GlyphXL" + }, + { + "id": "vae", + "required": false, + "label": "HunyuanVideo 1.5 VAE" + }, + { + "id": "tae", + "required": false, + "label": "TAEHV 1.5" + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "video_frames": 33, + "cfg_scale": 6, + "steps": 20, + "width": 1280, + "height": 720, + "flow_shift": 9 + }, + "features": [ + "flow_shift" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/ideogram4.json b/examples/server/recipes/ideogram4.json new file mode 100644 index 000000000..2d5230019 --- /dev/null +++ b/examples/server/recipes/ideogram4.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "id": "ideogram4", + "name": "Ideogram 4", + "description": "Select the matching unconditional transformer for classifier-free guidance.", + "model_url": "https://huggingface.co/ideogram-ai/ideogram-4-fp8", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "uncond_diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 8B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 7, + "steps": 20, + "width": 1024, + "height": 1024, + "sampling_method": "euler" + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes/instruct-pix2pix.json b/examples/server/recipes/instruct-pix2pix.json new file mode 100644 index 000000000..e81950910 --- /dev/null +++ b/examples/server/recipes/instruct-pix2pix.json @@ -0,0 +1,36 @@ +{ + "schema_version": 1, + "id": "instruct-pix2pix", + "name": "Instruct-Pix2Pix", + "description": "Supply the image to edit as a reference image.", + "model_url": "https://huggingface.co/timbrooks/instruct-pix2pix", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "img_cfg_scale": 1, + "sampling_method": "euler_a", + "scheduler": "karras", + "steps": 10 + }, + "features": [ + "cache_unet", + "clip_skip", + "img_cfg", + "ref_images" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/iris-3b.json b/examples/server/recipes/iris-3b.json new file mode 100644 index 000000000..c1b5d84fc --- /dev/null +++ b/examples/server/recipes/iris-3b.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "id": "iris-3b", + "name": "Iris 3B", + "description": "Pixel-space model; no VAE is required.", + "model_url": "https://huggingface.co/speridlabs/iris-3b", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 4B Text Encoder" + } + ], + "defaults": { + "cfg_scale": 3, + "steps": 20, + "width": 1024, + "height": 1024, + "sampling_method": "euler" + }, + "features": [ + "flow_shift" + ], + "disabled_features": [ + "preview", + "vae_tiling" + ] +} diff --git a/examples/server/recipes/krea2-raw-edit.json b/examples/server/recipes/krea2-raw-edit.json new file mode 100644 index 000000000..8d3eb9c7b --- /dev/null +++ b/examples/server/recipes/krea2-raw-edit.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "id": "krea2-raw-edit", + "name": "Krea 2 Raw Edit (LoRA)", + "description": "Requires a matching Ostris-compatible community editing LoRA. Apply it after loading and supply reference images; uses the krea2_ostris_edit preset. Select a separate vision projector when it is not bundled in the text encoder.", + "model_url": "https://huggingface.co/krea/Krea-2-Raw", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 4B Text Encoder" + }, + { + "id": "llm_vision", + "required": false, + "label": "Vision projector (required for separate GGUF text encoders)" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "scheduler": "discrete", + "sampling_method": "euler", + "cfg_scale": 3.5, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "flow_shift", + "ref_images" + ] +} diff --git a/examples/server/recipes/krea2-raw.json b/examples/server/recipes/krea2-raw.json new file mode 100644 index 000000000..5878291ab --- /dev/null +++ b/examples/server/recipes/krea2-raw.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "id": "krea2-raw", + "name": "Krea 2 Raw", + "description": "For reference-image editing with a community LoRA, use the corresponding Krea 2 Edit recipe.", + "model_url": "https://huggingface.co/krea/Krea-2-Raw", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 4B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 3.5, + "steps": 20, + "width": 1024, + "height": 1024, + "sampling_method": "euler" + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes/krea2-turbo-edit.json b/examples/server/recipes/krea2-turbo-edit.json new file mode 100644 index 000000000..fa248c083 --- /dev/null +++ b/examples/server/recipes/krea2-turbo-edit.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "id": "krea2-turbo-edit", + "name": "Krea 2 Turbo Edit (LoRA)", + "description": "Requires a matching Ostris-compatible community editing LoRA. Apply it after loading and supply reference images; uses the krea2_ostris_edit preset. Select a separate vision projector when it is not bundled in the text encoder.", + "model_url": "https://huggingface.co/krea/Krea-2-Turbo", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 4B Text Encoder" + }, + { + "id": "llm_vision", + "required": false, + "label": "Vision projector (required for separate GGUF text encoders)" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "scheduler": "discrete", + "sampling_method": "euler", + "cfg_scale": 1, + "steps": 8, + "width": 2048, + "height": 2048 + }, + "features": [ + "flow_shift", + "ref_images" + ] +} diff --git a/examples/server/recipes/krea2-turbo.json b/examples/server/recipes/krea2-turbo.json new file mode 100644 index 000000000..23dad12fd --- /dev/null +++ b/examples/server/recipes/krea2-turbo.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "id": "krea2-turbo", + "name": "Krea 2 Turbo", + "description": "For reference-image editing with a community LoRA, use the corresponding Krea 2 Edit recipe.", + "model_url": "https://huggingface.co/krea/Krea-2-Turbo", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 4B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 1, + "steps": 8, + "width": 2048, + "height": 2048, + "sampling_method": "euler" + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes/lens-turbo.json b/examples/server/recipes/lens-turbo.json new file mode 100644 index 000000000..e9282eafc --- /dev/null +++ b/examples/server/recipes/lens-turbo.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "id": "lens-turbo", + "name": "LENS Turbo", + "model_url": "https://huggingface.co/microsoft/Lens-Turbo", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "GPT-OSS 20B Text Encoder" + }, + { + "id": "tokenizer", + "required": true, + "label": "GPT-OSS tokenizer.json" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 1, + "steps": 4, + "width": 1440, + "height": 1440, + "sampling_method": "euler" + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes/lens.json b/examples/server/recipes/lens.json new file mode 100644 index 000000000..0a8954229 --- /dev/null +++ b/examples/server/recipes/lens.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "id": "lens", + "name": "LENS", + "model_url": "https://huggingface.co/microsoft/Lens", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "GPT-OSS 20B Text Encoder" + }, + { + "id": "tokenizer", + "required": true, + "label": "GPT-OSS tokenizer.json" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 5, + "steps": 20, + "width": 1440, + "height": 1440, + "sampling_method": "euler" + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes/lingbot-video-dense.json b/examples/server/recipes/lingbot-video-dense.json new file mode 100644 index 000000000..5f313a0cd --- /dev/null +++ b/examples/server/recipes/lingbot-video-dense.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "id": "lingbot-video-dense", + "name": "LingBot Video Dense 1.3B", + "description": "Use Video Generation.", + "model_url": "https://huggingface.co/robbyant/lingbot-video-dense-1.3b", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 4B Text Encoder" + }, + { + "id": "vae", + "required": false, + "label": "Wan 2.1 VAE" + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "video_frames": 33, + "cfg_scale": 3, + "steps": 20, + "width": 832, + "height": 480, + "sampling_method": "euler" + }, + "features": [ + "flow_shift", + "init_image" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/lingbot-video-moe.json b/examples/server/recipes/lingbot-video-moe.json new file mode 100644 index 000000000..cdbce61ee --- /dev/null +++ b/examples/server/recipes/lingbot-video-moe.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "id": "lingbot-video-moe", + "name": "LingBot Video MoE 30B A3B", + "description": "Use Video Generation.", + "model_url": "https://huggingface.co/robbyant/lingbot-video-moe-30b-a3b", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 4B Text Encoder" + }, + { + "id": "vae", + "required": false, + "label": "Wan 2.1 VAE" + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "video_frames": 33, + "cfg_scale": 3, + "steps": 20, + "width": 832, + "height": 480, + "sampling_method": "euler" + }, + "features": [ + "flow_shift", + "init_image" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/llada-image-turbo.json b/examples/server/recipes/llada-image-turbo.json new file mode 100644 index 000000000..0589bb384 --- /dev/null +++ b/examples/server/recipes/llada-image-turbo.json @@ -0,0 +1,52 @@ +{ + "schema_version": 1, + "id": "llada-image-turbo", + "name": "LLaDA Image Turbo", + "description": "Use matching turbo text encoder and connectors. Editing needs connectors that include the SigVQ encoder.", + "model_url": "https://huggingface.co/inclusionAI/LLaDA-Image-Turbo", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "LLaDA-Image Text Encoder" + }, + { + "id": "tokenizer", + "required": true, + "label": "LLaDA tokenizer.json" + }, + { + "id": "embeddings_connectors", + "required": true, + "label": "LLaDA Turbo Connectors" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 1, + "steps": 4, + "width": 1024, + "height": 1024, + "sampling_method": "euler" + }, + "features": [ + "ref_images" + ] +} diff --git a/examples/server/recipes/llada-image.json b/examples/server/recipes/llada-image.json new file mode 100644 index 000000000..27887c849 --- /dev/null +++ b/examples/server/recipes/llada-image.json @@ -0,0 +1,53 @@ +{ + "schema_version": 1, + "id": "llada-image", + "name": "LLaDA Image (base)", + "description": "Use matching base transformer, text encoder and connectors; do not mix them with Turbo components. Editing needs connectors with SigVQ and 1024x1024 output.", + "model_url": "https://huggingface.co/inclusionAI/LLaDA-Image", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "LLaDA-Image Base Text Encoder" + }, + { + "id": "tokenizer", + "required": true, + "label": "LLaDA tokenizer.json" + }, + { + "id": "embeddings_connectors", + "required": true, + "label": "LLaDA Base Connectors" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "scheduler": "llada_image", + "cfg_scale": 5, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "ref_images" + ] +} diff --git a/examples/server/recipes/longcat-image-edit.json b/examples/server/recipes/longcat-image-edit.json new file mode 100644 index 000000000..bc15ea2af --- /dev/null +++ b/examples/server/recipes/longcat-image-edit.json @@ -0,0 +1,43 @@ +{ + "schema_version": 1, + "id": "longcat-image-edit", + "name": "LongCat Image Edit", + "description": "Supply reference images for editing.", + "model_url": "https://huggingface.co/meituan-longcat/LongCat-Image-Edit", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 2.5 VL 7B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 4.5, + "steps": 20, + "sampling_method": "euler" + }, + "features": [ + "flow_shift", + "img_cfg", + "ref_images", + "slg" + ] +} diff --git a/examples/server/recipes/longcat-image.json b/examples/server/recipes/longcat-image.json new file mode 100644 index 000000000..36e23e85e --- /dev/null +++ b/examples/server/recipes/longcat-image.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "id": "longcat-image", + "name": "LongCat Image", + "model_url": "https://huggingface.co/meituan-longcat/LongCat-Image", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 2.5 VL 7B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 4, + "steps": 20, + "width": 1344, + "height": 768, + "sampling_method": "euler" + }, + "features": [ + "flow_shift", + "slg" + ] +} diff --git a/examples/server/recipes/ltx23-dev.json b/examples/server/recipes/ltx23-dev.json new file mode 100644 index 000000000..83bb934e8 --- /dev/null +++ b/examples/server/recipes/ltx23-dev.json @@ -0,0 +1,60 @@ +{ + "schema_version": 1, + "id": "ltx23-dev", + "name": "LTX 2.3 dev", + "description": "Supports text, start-image and start/end-image video. Audio VAE is optional for silent video. Configure latent upscaling separately in generation settings. Optional distillation LoRAs use this recipe; select the LoRA and adjust the sampling settings separately.", + "model_url": "https://huggingface.co/Lightricks/LTX-2.3", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Gemma 3 12B Text Encoder" + }, + { + "id": "embeddings_connectors", + "required": true + }, + { + "id": "vae", + "required": false, + "label": "LTX 2.3 Video VAE" + }, + { + "id": "tae", + "required": false + }, + { + "id": "audio_vae", + "required": false, + "label": "LTX 2.3 Audio VAE" + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "video_frames": 33, + "fps": 24, + "cfg_scale": 3, + "steps": 20, + "width": 768, + "height": 512 + }, + "features": [ + "end_image", + "flow_shift", + "init_image" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/ltx23-distilled.json b/examples/server/recipes/ltx23-distilled.json new file mode 100644 index 000000000..a2a6ac33c --- /dev/null +++ b/examples/server/recipes/ltx23-distilled.json @@ -0,0 +1,60 @@ +{ + "schema_version": 1, + "id": "ltx23-distilled", + "name": "LTX 2.3 Distilled 1.1", + "description": "Supports text, start-image and start/end-image video. Audio VAE is optional for silent video. Configure latent upscaling separately in generation settings.", + "model_url": "https://huggingface.co/Lightricks/LTX-2.3", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Gemma 3 12B Text Encoder" + }, + { + "id": "embeddings_connectors", + "required": true + }, + { + "id": "vae", + "required": false, + "label": "LTX 2.3 Video VAE" + }, + { + "id": "tae", + "required": false + }, + { + "id": "audio_vae", + "required": false, + "label": "LTX 2.3 Audio VAE" + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "video_frames": 121, + "fps": 24, + "cfg_scale": 1, + "steps": 8, + "width": 768, + "height": 512 + }, + "features": [ + "end_image", + "flow_shift", + "init_image" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/ltx25-dev.json b/examples/server/recipes/ltx25-dev.json new file mode 100644 index 000000000..47efbc76f --- /dev/null +++ b/examples/server/recipes/ltx25-dev.json @@ -0,0 +1,56 @@ +{ + "schema_version": 1, + "id": "ltx25-dev", + "name": "LTX 2.5 dev", + "description": "Use the conv video VAE and LTX-specific Gemma 4 with bundled projection. Audio VAE is optional for silent video.", + "model_url": "https://huggingface.co/Lightricks/LTX-2.5", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "LTX-2.5 Gemma 4 12B with projection" + }, + { + "id": "vae", + "required": false, + "label": "LTX 2.5 Conv Video VAE" + }, + { + "id": "tae", + "required": false + }, + { + "id": "audio_vae", + "required": false, + "label": "LTX 2.5 Audio VAE" + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "fps": 24, + "video_frames": 33, + "cfg_scale": 3, + "steps": 20, + "width": 768, + "height": 512 + }, + "features": [ + "end_image", + "flow_shift", + "init_image" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/mage-flow-edit.json b/examples/server/recipes/mage-flow-edit.json new file mode 100644 index 000000000..82d66f6bd --- /dev/null +++ b/examples/server/recipes/mage-flow-edit.json @@ -0,0 +1,39 @@ +{ + "schema_version": 1, + "id": "mage-flow-edit", + "name": "Mage Flow Edit", + "description": "Supply reference images. Select a separate vision projector when it is not bundled in the text encoder.", + "model_url": "https://huggingface.co/microsoft/Mage-Flow-Edit", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 4B Text Encoder" + }, + { + "id": "vae", + "required": true + }, + { + "id": "llm_vision", + "required": false, + "label": "Vision projector (required for separate GGUF text encoders)" + } + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 5, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "flow_shift", + "img_cfg", + "ref_images" + ] +} diff --git a/examples/server/recipes/mage-flow-turbo.json b/examples/server/recipes/mage-flow-turbo.json new file mode 100644 index 000000000..5cf9bb25a --- /dev/null +++ b/examples/server/recipes/mage-flow-turbo.json @@ -0,0 +1,31 @@ +{ + "schema_version": 1, + "id": "mage-flow-turbo", + "name": "Mage Flow Turbo", + "model_url": "https://huggingface.co/microsoft/Mage-Flow-Turbo", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 4B Text Encoder" + }, + { + "id": "vae", + "required": true + } + ], + "defaults": { + "cfg_scale": 1, + "steps": 4, + "width": 1024, + "height": 1024, + "sampling_method": "euler" + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes/mage-flow.json b/examples/server/recipes/mage-flow.json new file mode 100644 index 000000000..4cbd15934 --- /dev/null +++ b/examples/server/recipes/mage-flow.json @@ -0,0 +1,31 @@ +{ + "schema_version": 1, + "id": "mage-flow", + "name": "Mage Flow", + "model_url": "https://huggingface.co/microsoft/Mage-Flow", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 4B Text Encoder" + }, + { + "id": "vae", + "required": true + } + ], + "defaults": { + "cfg_scale": 5, + "steps": 20, + "width": 1024, + "height": 1024, + "sampling_method": "euler" + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes/ming-image.json b/examples/server/recipes/ming-image.json new file mode 100644 index 000000000..6c368e8e1 --- /dev/null +++ b/examples/server/recipes/ming-image.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "id": "ming-image", + "name": "Ming Image 0.1", + "model_url": "https://huggingface.co/inclusionAI/Ming-Image-0.1-Design", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Ling Mini 2.0 Text Encoder" + }, + { + "id": "tokenizer", + "required": true, + "label": "Ling tokenizer.json" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 1, + "steps": 12, + "width": 2048, + "height": 2048, + "sampling_method": "euler" + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes/minimax-h3-fl2va.json b/examples/server/recipes/minimax-h3-fl2va.json new file mode 100644 index 000000000..397192391 --- /dev/null +++ b/examples/server/recipes/minimax-h3-fl2va.json @@ -0,0 +1,55 @@ +{ + "schema_version": 1, + "id": "minimax-h3-fl2va", + "name": "MiniMax H3 FL2VA", + "description": "Generate video and audio, optionally with start/end images. Supports full and pruned FL2VA weights. Audio VAE is optional for silent video.", + "model_url": "https://huggingface.co/MiniMaxAI/MiniMax-H3", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "MiniMax-H3 Qwen 3 VL 32B Text Encoder" + }, + { + "id": "vae", + "required": false, + "label": "MiniMax H3 Video VAE" + }, + { + "id": "tae", + "required": false + }, + { + "id": "audio_vae", + "required": false, + "label": "MiniMax H3 Audio VAE" + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "video_frames": 56, + "fps": 24, + "cfg_scale": 1, + "width": 1344, + "height": 768, + "sampling_method": "euler" + }, + "features": [ + "end_image", + "flow_shift", + "init_image" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/minimax-h3-ref2va.json b/examples/server/recipes/minimax-h3-ref2va.json new file mode 100644 index 000000000..085f5d5fe --- /dev/null +++ b/examples/server/recipes/minimax-h3-ref2va.json @@ -0,0 +1,54 @@ +{ + "schema_version": 1, + "id": "minimax-h3-ref2va", + "name": "MiniMax H3 Ref2VA", + "description": "Supply image, video or audio references with compatible Ref2VA weights. Audio VAE is optional for silent video.", + "model_url": "https://huggingface.co/MiniMaxAI/MiniMax-H3", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "MiniMax-H3 Qwen 3 VL 32B Text Encoder" + }, + { + "id": "vae", + "required": false, + "label": "MiniMax H3 Video VAE" + }, + { + "id": "tae", + "required": false + }, + { + "id": "audio_vae", + "required": false, + "label": "MiniMax H3 Audio VAE" + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "video_frames": 56, + "fps": 24, + "cfg_scale": 1, + "width": 1344, + "height": 768, + "sampling_method": "euler" + }, + "features": [ + "flow_shift", + "ref_images" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/minit2i.json b/examples/server/recipes/minit2i.json new file mode 100644 index 000000000..2c7b94eba --- /dev/null +++ b/examples/server/recipes/minit2i.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "id": "minit2i", + "name": "MiniT2I", + "description": "Pixel-space model; no VAE is required.", + "model_url": "https://huggingface.co/MiniT2I/MiniT2I", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "t5xxl", + "required": true, + "label": "FLAN-T5 Large" + } + ], + "defaults": { + "cfg_scale": 2.5, + "steps": 20, + "width": 512, + "height": 512, + "sampling_method": "euler" + }, + "disabled_features": [ + "preview", + "vae_tiling" + ] +} diff --git a/examples/server/recipes/moss-image-base.json b/examples/server/recipes/moss-image-base.json new file mode 100644 index 000000000..bc2cc7d9b --- /dev/null +++ b/examples/server/recipes/moss-image-base.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "id": "moss-image-base", + "name": "MOSS Image Base", + "description": "Select the MOSS-Image-Base transformer with its matching text encoder and FLUX.2 VAE or TAE.", + "model_url": "https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3.5 9B Text Encoder" + }, + { + "id": "tokenizer", + "required": true, + "label": "Qwen 3.5 tokenizer.json" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 3, + "steps": 20, + "width": 1024, + "height": 1024, + "flow_shift": 5, + "sampling_method": "euler" + }, + "features": [ + "flow_shift", + "slg" + ] +} diff --git a/examples/server/recipes/moss-image-sft.json b/examples/server/recipes/moss-image-sft.json new file mode 100644 index 000000000..e0239b0bc --- /dev/null +++ b/examples/server/recipes/moss-image-sft.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "id": "moss-image-sft", + "name": "MOSS Image SFT", + "description": "Select the MOSS-Image-SFT transformer with its matching text encoder and FLUX.2 VAE or TAE.", + "model_url": "https://huggingface.co/OpenMOSS-Team/MOSS-Image-SFT", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3.5 9B Text Encoder" + }, + { + "id": "tokenizer", + "required": true, + "label": "Qwen 3.5 tokenizer.json" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 3, + "steps": 20, + "width": 1024, + "height": 1024, + "flow_shift": 5, + "sampling_method": "euler" + }, + "features": [ + "flow_shift", + "slg" + ] +} diff --git a/examples/server/recipes/nitrosd-realism.json b/examples/server/recipes/nitrosd-realism.json new file mode 100644 index 000000000..75f2d43ff --- /dev/null +++ b/examples/server/recipes/nitrosd-realism.json @@ -0,0 +1,36 @@ +{ + "schema_version": 1, + "id": "nitrosd-realism", + "name": "NitroSD Realism", + "model_url": "https://huggingface.co/ChenDY/NitroFusion", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "scheduler": "sgm_uniform", + "shifted_timestep": 250, + "cfg_scale": 1, + "steps": 1, + "width": 1024, + "height": 1024 + }, + "features": [ + "cache_unet", + "clip_skip", + "shifted_timestep" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/ovis-image.json b/examples/server/recipes/ovis-image.json new file mode 100644 index 000000000..21e9d9fce --- /dev/null +++ b/examples/server/recipes/ovis-image.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "id": "ovis-image", + "name": "Ovis Image", + "model_url": "https://huggingface.co/ATH-MaaS/Ovis-Image-7B", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Ovis 2.5 Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 5, + "steps": 20, + "width": 1024, + "height": 1024, + "sampling_method": "euler" + }, + "features": [ + "flow_shift", + "slg" + ] +} diff --git a/examples/server/recipes/pid-15-flux2.json b/examples/server/recipes/pid-15-flux2.json new file mode 100644 index 000000000..4fb3db3a7 --- /dev/null +++ b/examples/server/recipes/pid-15-flux2.json @@ -0,0 +1,47 @@ +{ + "schema_version": 1, + "id": "pid-15-flux2", + "name": "PiD 1.5 (Flux.2 VAE)", + "description": "Supply a reference image; the first image is used as the source latent. Select a matching Flux.2 PiD checkpoint and VAE.", + "model_url": "https://huggingface.co/nvidia/PiD", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Gemma 2 2B ELM Text Encoder" + }, + { + "id": "tokenizer", + "required": true, + "label": "Gemma 2 tokenizer.json" + }, + { + "id": "vae", + "required": true + } + ], + "context": { + "vae_format": "flux2" + }, + "defaults": { + "sampling_method": "lcm", + "scheduler": "lcm", + "cfg_scale": 1, + "steps": 4, + "width": 2048, + "height": 2048 + }, + "features": [ + "flow_shift", + "ref_images" + ], + "disabled_features": [ + "img2img", + "mask_image", + "preview" + ] +} diff --git a/examples/server/recipes/pid-15-qwen-image.json b/examples/server/recipes/pid-15-qwen-image.json new file mode 100644 index 000000000..9db9e7422 --- /dev/null +++ b/examples/server/recipes/pid-15-qwen-image.json @@ -0,0 +1,47 @@ +{ + "schema_version": 1, + "id": "pid-15-qwen-image", + "name": "PiD 1.5 (Qwen Image VAE)", + "description": "Supply a reference image; the first image is used as the source latent. Select a matching Qwen Image PiD checkpoint and VAE. Use the Qwen Image 2D VAE.", + "model_url": "https://huggingface.co/nvidia/PiD", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Gemma 2 2B ELM Text Encoder" + }, + { + "id": "tokenizer", + "required": true, + "label": "Gemma 2 tokenizer.json" + }, + { + "id": "vae", + "required": true + } + ], + "context": { + "vae_format": "wan" + }, + "defaults": { + "sampling_method": "lcm", + "scheduler": "lcm", + "cfg_scale": 1, + "steps": 4, + "width": 2048, + "height": 2048 + }, + "features": [ + "flow_shift", + "ref_images" + ], + "disabled_features": [ + "img2img", + "mask_image", + "preview" + ] +} diff --git a/examples/server/recipes/pid-15-sd3.json b/examples/server/recipes/pid-15-sd3.json new file mode 100644 index 000000000..3980b8671 --- /dev/null +++ b/examples/server/recipes/pid-15-sd3.json @@ -0,0 +1,47 @@ +{ + "schema_version": 1, + "id": "pid-15-sd3", + "name": "PiD 1.5 (SD3 VAE)", + "description": "Supply a reference image; the first image is used as the source latent. Select a matching SD3 PiD checkpoint and VAE.", + "model_url": "https://huggingface.co/nvidia/PiD", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Gemma 2 2B ELM Text Encoder" + }, + { + "id": "tokenizer", + "required": true, + "label": "Gemma 2 tokenizer.json" + }, + { + "id": "vae", + "required": true + } + ], + "context": { + "vae_format": "sd3" + }, + "defaults": { + "sampling_method": "lcm", + "scheduler": "lcm", + "cfg_scale": 1, + "steps": 4, + "width": 2048, + "height": 2048 + }, + "features": [ + "flow_shift", + "ref_images" + ], + "disabled_features": [ + "img2img", + "mask_image", + "preview" + ] +} diff --git a/examples/server/recipes/pid-15.json b/examples/server/recipes/pid-15.json new file mode 100644 index 000000000..f7913bee8 --- /dev/null +++ b/examples/server/recipes/pid-15.json @@ -0,0 +1,47 @@ +{ + "schema_version": 1, + "id": "pid-15", + "name": "PiD 1.5 (Flux / Z-Image VAE)", + "description": "Supply a reference image; the first image is used as the source latent. Select a Flux/Z-Image PiD checkpoint and matching Flux VAE.", + "model_url": "https://huggingface.co/nvidia/PiD", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Gemma 2 2B ELM Text Encoder" + }, + { + "id": "tokenizer", + "required": true, + "label": "Gemma 2 tokenizer.json" + }, + { + "id": "vae", + "required": true + } + ], + "context": { + "vae_format": "flux" + }, + "defaults": { + "sampling_method": "lcm", + "scheduler": "lcm", + "cfg_scale": 1, + "steps": 4, + "width": 2048, + "height": 2048 + }, + "features": [ + "flow_shift", + "ref_images" + ], + "disabled_features": [ + "img2img", + "mask_image", + "preview" + ] +} diff --git a/examples/server/recipes/pid-flux2.json b/examples/server/recipes/pid-flux2.json new file mode 100644 index 000000000..9c802dc15 --- /dev/null +++ b/examples/server/recipes/pid-flux2.json @@ -0,0 +1,47 @@ +{ + "schema_version": 1, + "id": "pid-flux2", + "name": "PiD (Flux.2 VAE)", + "description": "Supply a reference image; the first image is used as the source latent. Select a matching Flux.2 PiD checkpoint and VAE.", + "model_url": "https://huggingface.co/nvidia/PiD", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Gemma 2 2B ELM Text Encoder" + }, + { + "id": "tokenizer", + "required": true, + "label": "Gemma 2 tokenizer.json" + }, + { + "id": "vae", + "required": true + } + ], + "context": { + "vae_format": "flux2" + }, + "defaults": { + "sampling_method": "lcm", + "scheduler": "lcm", + "cfg_scale": 1, + "steps": 4, + "width": 2048, + "height": 2048 + }, + "features": [ + "flow_shift", + "ref_images" + ], + "disabled_features": [ + "img2img", + "mask_image", + "preview" + ] +} diff --git a/examples/server/recipes/pid-qwen-image.json b/examples/server/recipes/pid-qwen-image.json new file mode 100644 index 000000000..4503b1cd6 --- /dev/null +++ b/examples/server/recipes/pid-qwen-image.json @@ -0,0 +1,47 @@ +{ + "schema_version": 1, + "id": "pid-qwen-image", + "name": "PiD (Qwen Image VAE)", + "description": "Supply a reference image; the first image is used as the source latent. Select a matching Qwen Image PiD checkpoint and VAE. Use the Qwen Image 2D VAE.", + "model_url": "https://huggingface.co/nvidia/PiD", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Gemma 2 2B ELM Text Encoder" + }, + { + "id": "tokenizer", + "required": true, + "label": "Gemma 2 tokenizer.json" + }, + { + "id": "vae", + "required": true + } + ], + "context": { + "vae_format": "wan" + }, + "defaults": { + "sampling_method": "lcm", + "scheduler": "lcm", + "cfg_scale": 1, + "steps": 4, + "width": 2048, + "height": 2048 + }, + "features": [ + "flow_shift", + "ref_images" + ], + "disabled_features": [ + "img2img", + "mask_image", + "preview" + ] +} diff --git a/examples/server/recipes/pid-sd3.json b/examples/server/recipes/pid-sd3.json new file mode 100644 index 000000000..6dad2ca09 --- /dev/null +++ b/examples/server/recipes/pid-sd3.json @@ -0,0 +1,47 @@ +{ + "schema_version": 1, + "id": "pid-sd3", + "name": "PiD (SD3 VAE)", + "description": "Supply a reference image; the first image is used as the source latent. Select a matching SD3 PiD checkpoint and VAE.", + "model_url": "https://huggingface.co/nvidia/PiD", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Gemma 2 2B ELM Text Encoder" + }, + { + "id": "tokenizer", + "required": true, + "label": "Gemma 2 tokenizer.json" + }, + { + "id": "vae", + "required": true + } + ], + "context": { + "vae_format": "sd3" + }, + "defaults": { + "sampling_method": "lcm", + "scheduler": "lcm", + "cfg_scale": 1, + "steps": 4, + "width": 2048, + "height": 2048 + }, + "features": [ + "flow_shift", + "ref_images" + ], + "disabled_features": [ + "img2img", + "mask_image", + "preview" + ] +} diff --git a/examples/server/recipes/pid.json b/examples/server/recipes/pid.json new file mode 100644 index 000000000..247944965 --- /dev/null +++ b/examples/server/recipes/pid.json @@ -0,0 +1,47 @@ +{ + "schema_version": 1, + "id": "pid", + "name": "PiD (Flux / Z-Image VAE)", + "description": "Supply a reference image; the first image is used as the source latent. Select a Flux/Z-Image PiD checkpoint and matching Flux VAE.", + "model_url": "https://huggingface.co/nvidia/PiD", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Gemma 2 2B ELM Text Encoder" + }, + { + "id": "tokenizer", + "required": true, + "label": "Gemma 2 tokenizer.json" + }, + { + "id": "vae", + "required": true + } + ], + "context": { + "vae_format": "flux" + }, + "defaults": { + "sampling_method": "lcm", + "scheduler": "lcm", + "cfg_scale": 1, + "steps": 4, + "width": 2048, + "height": 2048 + }, + "features": [ + "flow_shift", + "ref_images" + ], + "disabled_features": [ + "img2img", + "mask_image", + "preview" + ] +} diff --git a/examples/server/recipes/pixart-alpha-1024.json b/examples/server/recipes/pixart-alpha-1024.json new file mode 100644 index 000000000..1144f87b2 --- /dev/null +++ b/examples/server/recipes/pixart-alpha-1024.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "id": "pixart-alpha-1024", + "name": "PixArt Alpha XL 1024", + "description": "Use the matching PixArt Alpha transformer and SD 1.x-style VAE. Resolution/aspect-ratio micro-conditioning is not currently applied by the runtime.", + "model_url": "https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "t5xxl", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "context": { + "model_args": "pixart_vae_scale_factor=0.18215" + }, + "defaults": { + "scheduler": "discrete", + "sampling_method": "euler", + "cfg_scale": 4.5, + "steps": 20, + "width": 1024, + "height": 1024 + } +} diff --git a/examples/server/recipes/pixart-alpha-512.json b/examples/server/recipes/pixart-alpha-512.json new file mode 100644 index 000000000..804a7a0c9 --- /dev/null +++ b/examples/server/recipes/pixart-alpha-512.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "id": "pixart-alpha-512", + "name": "PixArt Alpha XL 512", + "description": "Use the matching PixArt Alpha transformer and SD 1.x-style VAE. Resolution/aspect-ratio micro-conditioning is not currently applied by the runtime.", + "model_url": "https://huggingface.co/PixArt-alpha/PixArt-XL-2-512x512", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "t5xxl", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "context": { + "model_args": "pixart_vae_scale_factor=0.18215,pixart_pos_embed_base_size=32,pixart_interpolation_scale=1" + }, + "defaults": { + "scheduler": "discrete", + "sampling_method": "euler", + "cfg_scale": 4.5, + "steps": 20, + "width": 512, + "height": 512 + } +} diff --git a/examples/server/recipes/pixart-sigma.json b/examples/server/recipes/pixart-sigma.json new file mode 100644 index 000000000..ed4d2f0de --- /dev/null +++ b/examples/server/recipes/pixart-sigma.json @@ -0,0 +1,37 @@ +{ + "schema_version": 1, + "id": "pixart-sigma", + "name": "PixArt Sigma XL 1024", + "model_url": "https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "t5xxl", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 4.5, + "steps": 20, + "width": 1024, + "height": 1024, + "sampling_method": "euler" + } +} diff --git a/examples/server/recipes/qwen-image-21-turbo.json b/examples/server/recipes/qwen-image-21-turbo.json new file mode 100644 index 000000000..2ef9fa183 --- /dev/null +++ b/examples/server/recipes/qwen-image-21-turbo.json @@ -0,0 +1,60 @@ +{ + "schema_version": 1, + "id": "qwen-image-21-turbo", + "name": "Qwen Image 2.1 Turbo", + "description": "Use the standalone Turbo diffusion weights for generation and editing with the fixed 8-step schedule. Select the Qwen Image 2.1 VAE or TAEQI2.1, and supply a vision projector when needed.", + "model_url": "https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 8B Text Encoder" + }, + { + "id": "llm_vision", + "required": false, + "label": "Vision projector (required for separate GGUF text encoders)" + }, + { + "id": "vae", + "required": false, + "label": "Qwen Image 2.1 VAE" + }, + { + "id": "tae", + "required": false, + "label": "TAEQI2.1" + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 1, + "steps": 8, + "width": 2048, + "height": 2048, + "custom_sigmas": [ + 1.0, + 0.978453, + 0.95418, + 0.926626, + 0.89508, + 0.845148, + 0.704534, + 0.414568, + 0.0 + ] + }, + "features": [ + "ref_images" + ] +} diff --git a/examples/server/recipes/qwen-image-21.json b/examples/server/recipes/qwen-image-21.json new file mode 100644 index 000000000..4b6c67b86 --- /dev/null +++ b/examples/server/recipes/qwen-image-21.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "id": "qwen-image-21", + "name": "Qwen Image 2.1", + "description": "Supports generation and editing. Select the full Qwen Image 2.1 VAE or TAEQI2.1. Supply a separate vision projector when needed. Select acceleration LoRAs and adjust their settings separately.", + "model_url": "https://huggingface.co/Qwen/Qwen-Image-2.1", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 8B Text Encoder" + }, + { + "id": "llm_vision", + "required": false, + "label": "Vision projector (required for separate GGUF text encoders)" + }, + { + "id": "vae", + "required": false, + "label": "Qwen Image 2.1 VAE" + }, + { + "id": "tae", + "required": false, + "label": "TAEQI2.1" + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 1, + "steps": 20, + "width": 2048, + "height": 2048 + }, + "features": [ + "flow_shift", + "ref_images" + ] +} diff --git a/examples/server/recipes/qwen-image-edit-2509.json b/examples/server/recipes/qwen-image-edit-2509.json new file mode 100644 index 000000000..4c647a042 --- /dev/null +++ b/examples/server/recipes/qwen-image-edit-2509.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "id": "qwen-image-edit-2509", + "name": "Qwen Image Edit 2509", + "description": "Supply reference images. Select a vision projector for GGUF text encoders; omit it if vision weights are included in the text encoder.", + "model_url": "https://huggingface.co/Qwen/Qwen-Image-Edit-2509", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 2.5 VL 7B Text Encoder" + }, + { + "id": "llm_vision", + "required": false, + "label": "Vision projector (required for separate GGUF text encoders)" + }, + { + "id": "vae", + "required": false, + "label": "Qwen Image VAE" + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "flow_shift": 3.0, + "cfg_scale": 4, + "steps": 20 + }, + "features": [ + "flow_shift", + "img_cfg", + "ref_images" + ] +} diff --git a/examples/server/recipes/qwen-image-edit-2511.json b/examples/server/recipes/qwen-image-edit-2511.json new file mode 100644 index 000000000..cdd16db6c --- /dev/null +++ b/examples/server/recipes/qwen-image-edit-2511.json @@ -0,0 +1,52 @@ +{ + "schema_version": 1, + "id": "qwen-image-edit-2511", + "name": "Qwen Image Edit 2511", + "description": "Supply reference images. Select a vision projector for GGUF text encoders; omit it if vision weights are included in the text encoder.", + "model_url": "https://huggingface.co/Qwen/Qwen-Image-Edit-2511", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 2.5 VL 7B Text Encoder" + }, + { + "id": "llm_vision", + "required": false, + "label": "Vision projector (required for separate GGUF text encoders)" + }, + { + "id": "vae", + "required": false, + "label": "Qwen Image VAE" + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "context": { + "model_args": "qwen_image_zero_cond_t=true" + }, + "defaults": { + "sampling_method": "euler", + "flow_shift": 3.0, + "cfg_scale": 4, + "steps": 20 + }, + "features": [ + "flow_shift", + "img_cfg", + "ref_images" + ] +} diff --git a/examples/server/recipes/qwen-image-edit.json b/examples/server/recipes/qwen-image-edit.json new file mode 100644 index 000000000..5610a6c42 --- /dev/null +++ b/examples/server/recipes/qwen-image-edit.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "id": "qwen-image-edit", + "name": "Qwen Image Edit", + "description": "Supply reference images. Select a vision projector for GGUF text encoders; omit it if vision weights are included in the text encoder.", + "model_url": "https://huggingface.co/Qwen/Qwen-Image-Edit", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 2.5 VL 7B Text Encoder" + }, + { + "id": "llm_vision", + "required": false, + "label": "Vision projector (required for separate GGUF text encoders)" + }, + { + "id": "vae", + "required": false, + "label": "Qwen Image VAE" + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "flow_shift": 3.0, + "cfg_scale": 4, + "steps": 20 + }, + "features": [ + "flow_shift", + "img_cfg", + "ref_images" + ] +} diff --git a/examples/server/recipes/qwen-image-layered.json b/examples/server/recipes/qwen-image-layered.json new file mode 100644 index 000000000..96863be31 --- /dev/null +++ b/examples/server/recipes/qwen-image-layered.json @@ -0,0 +1,36 @@ +{ + "schema_version": 1, + "id": "qwen-image-layered", + "name": "Qwen Image Layered", + "description": "Supply a reference image and use the dedicated layered VAE.", + "model_url": "https://huggingface.co/Qwen/Qwen-Image-Layered", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 2.5 VL 7B Text Encoder" + }, + { + "id": "vae", + "required": true, + "label": "Qwen Image Layered VAE" + } + ], + "defaults": { + "sampling_method": "euler", + "flow_shift": 3.0, + "cfg_scale": 4, + "steps": 20, + "width": 640, + "height": 640 + }, + "features": [ + "flow_shift", + "img_cfg", + "ref_images" + ] +} diff --git a/examples/server/recipes/qwen-image.json b/examples/server/recipes/qwen-image.json new file mode 100644 index 000000000..417a621ae --- /dev/null +++ b/examples/server/recipes/qwen-image.json @@ -0,0 +1,44 @@ +{ + "schema_version": 1, + "id": "qwen-image", + "name": "Qwen Image", + "description": "Supports text-to-image and image-to-image; pruned transformer variants use the same components.", + "model_url": "https://huggingface.co/Qwen/Qwen-Image", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 2.5 VL 7B Text Encoder" + }, + { + "id": "vae", + "required": false, + "label": "Qwen Image VAE" + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "flow_shift": 3.0, + "cfg_scale": 4, + "steps": 20, + "width": 1328, + "height": 1328 + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes/sd-turbo.json b/examples/server/recipes/sd-turbo.json new file mode 100644 index 000000000..7723c8107 --- /dev/null +++ b/examples/server/recipes/sd-turbo.json @@ -0,0 +1,36 @@ +{ + "schema_version": 1, + "id": "sd-turbo", + "name": "SD Turbo", + "description": "Select a single-file SD-Turbo checkpoint. Uses one step with CFG disabled (scale 1 in sd.cpp); use more steps for partial image-to-image denoising.", + "model_url": "https://huggingface.co/stabilityai/sd-turbo", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "sampling_method": "euler", + "scheduler": "discrete", + "cfg_scale": 1, + "steps": 1, + "width": 512, + "height": 512 + }, + "features": [ + "cache_unet", + "clip_skip" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/sd1-checkpoint.json b/examples/server/recipes/sd1-checkpoint.json new file mode 100644 index 000000000..38d238677 --- /dev/null +++ b/examples/server/recipes/sd1-checkpoint.json @@ -0,0 +1,35 @@ +{ + "schema_version": 1, + "id": "sd1-checkpoint", + "name": "SD 1.x", + "description": "Supports SD 1.x, compatible LCM checkpoints and layer-count variants. For LCM weights, select LCM sampling and adjust steps/CFG. Tiny SD Diffusers directories must first be converted into a single-file checkpoint.", + "model_url": "https://huggingface.co/CompVis/stable-diffusion-v1-4", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "vae", + "label": "VAE override", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "cfg_scale": 7.5, + "steps": 20, + "width": 512, + "height": 512 + }, + "features": [ + "cache_unet", + "clip_skip" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/sd1-controlnet.json b/examples/server/recipes/sd1-controlnet.json new file mode 100644 index 000000000..eeaee22fb --- /dev/null +++ b/examples/server/recipes/sd1-controlnet.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "id": "sd1-controlnet", + "name": "SD 1.5 ControlNet", + "description": "Supply a control image; input image and mask are optional for image-to-image or inpainting.", + "model_url": "https://huggingface.co/lllyasviel/control_v11p_sd15_canny", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "control_net", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "cfg_scale": 7.5, + "steps": 20, + "width": 512, + "height": 512 + }, + "features": [ + "cache_unet", + "clip_skip" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/sd1-inpainting.json b/examples/server/recipes/sd1-inpainting.json new file mode 100644 index 000000000..f9f3064f3 --- /dev/null +++ b/examples/server/recipes/sd1-inpainting.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "id": "sd1-inpainting", + "name": "SD 1.5 Inpainting", + "description": "Select an inpainting checkpoint, then supply an input image and mask.", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "cfg_scale": 7.5, + "steps": 20, + "width": 512, + "height": 512 + }, + "features": [ + "cache_unet", + "clip_skip", + "img_cfg" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/sd1-ip-adapter.json b/examples/server/recipes/sd1-ip-adapter.json new file mode 100644 index 000000000..6c0ab1a6b --- /dev/null +++ b/examples/server/recipes/sd1-ip-adapter.json @@ -0,0 +1,43 @@ +{ + "schema_version": 1, + "id": "sd1-ip-adapter", + "name": "SD 1.5 IP-Adapter / Plus", + "description": "Select matching SD 1.5 IP-Adapter or Plus weights and CLIP Vision H, then supply an IP-Adapter image.", + "model_url": "https://huggingface.co/h94/IP-Adapter", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "clip_vision", + "required": true, + "label": "CLIP Vision H" + }, + { + "id": "ip_adapter", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "cfg_scale": 7.5, + "steps": 20, + "width": 512, + "height": 512 + }, + "features": [ + "cache_unet", + "clip_skip" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/sd2-512-checkpoint.json b/examples/server/recipes/sd2-512-checkpoint.json new file mode 100644 index 000000000..77838c280 --- /dev/null +++ b/examples/server/recipes/sd2-512-checkpoint.json @@ -0,0 +1,33 @@ +{ + "schema_version": 1, + "id": "sd2-512-checkpoint", + "name": "SD 2.1 512", + "description": "Supports SD 2.x 512 and compatible layer-count variants. Tiny SD2 Diffusers directories must first be converted into a single-file checkpoint.", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "cfg_scale": 7.5, + "steps": 20, + "width": 512, + "height": 512 + }, + "features": [ + "cache_unet", + "clip_skip" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/sd2-768-checkpoint.json b/examples/server/recipes/sd2-768-checkpoint.json new file mode 100644 index 000000000..808b187d7 --- /dev/null +++ b/examples/server/recipes/sd2-768-checkpoint.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "id": "sd2-768-checkpoint", + "name": "SD 2.1 768", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "cfg_scale": 7.5, + "steps": 20, + "width": 768, + "height": 768 + }, + "features": [ + "cache_unet", + "clip_skip" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/sd2-inpainting.json b/examples/server/recipes/sd2-inpainting.json new file mode 100644 index 000000000..4980f7f0a --- /dev/null +++ b/examples/server/recipes/sd2-inpainting.json @@ -0,0 +1,36 @@ +{ + "schema_version": 1, + "id": "sd2-inpainting", + "name": "SD 2 Inpainting", + "description": "Select an inpainting checkpoint, then supply Init Image and Mask Image.", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "scheduler": "discrete", + "sampling_method": "euler_a", + "cfg_scale": 7.5, + "steps": 20, + "width": 512, + "height": 512 + }, + "features": [ + "cache_unet", + "clip_skip", + "img_cfg" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/sd3-medium.json b/examples/server/recipes/sd3-medium.json new file mode 100644 index 000000000..7c0e8fbff --- /dev/null +++ b/examples/server/recipes/sd3-medium.json @@ -0,0 +1,33 @@ +{ + "schema_version": 1, + "id": "sd3-medium", + "name": "SD 3 Medium", + "description": "Checkpoint with CLIP-L, CLIP-G and T5 included.", + "model_url": "https://huggingface.co/stabilityai/stable-diffusion-3-medium-diffusers", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 7, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "clip_skip", + "flow_shift", + "slg" + ] +} diff --git a/examples/server/recipes/sd35-large.json b/examples/server/recipes/sd35-large.json new file mode 100644 index 000000000..a231db99c --- /dev/null +++ b/examples/server/recipes/sd35-large.json @@ -0,0 +1,44 @@ +{ + "schema_version": 1, + "id": "sd35-large", + "name": "SD 3.5 Large", + "model_url": "https://huggingface.co/stabilityai/stable-diffusion-3.5-large", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "clip_l", + "required": true + }, + { + "id": "clip_g", + "required": true + }, + { + "id": "t5xxl", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 3.5, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "clip_skip", + "flow_shift", + "slg" + ] +} diff --git a/examples/server/recipes/sd35-medium.json b/examples/server/recipes/sd35-medium.json new file mode 100644 index 000000000..75080253a --- /dev/null +++ b/examples/server/recipes/sd35-medium.json @@ -0,0 +1,44 @@ +{ + "schema_version": 1, + "id": "sd35-medium", + "name": "SD 3.5 Medium", + "model_url": "https://huggingface.co/stabilityai/stable-diffusion-3.5-medium", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "clip_l", + "required": true + }, + { + "id": "clip_g", + "required": true + }, + { + "id": "t5xxl", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 4.5, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "clip_skip", + "flow_shift", + "slg" + ] +} diff --git a/examples/server/recipes/sdxl-checkpoint.json b/examples/server/recipes/sdxl-checkpoint.json new file mode 100644 index 000000000..372d5a6d4 --- /dev/null +++ b/examples/server/recipes/sdxl-checkpoint.json @@ -0,0 +1,36 @@ +{ + "schema_version": 1, + "id": "sdxl-checkpoint", + "name": "SDXL", + "description": "Supports SDXL and compatible reduced-depth checkpoints such as SSD-1B and Segmind Vega.", + "model_url": "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "vae", + "label": "VAE override", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "cfg_scale": 5, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "cache_unet", + "clip_skip", + "shifted_timestep" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/sdxl-controlnet.json b/examples/server/recipes/sdxl-controlnet.json new file mode 100644 index 000000000..c50efbd2b --- /dev/null +++ b/examples/server/recipes/sdxl-controlnet.json @@ -0,0 +1,41 @@ +{ + "schema_version": 1, + "id": "sdxl-controlnet", + "name": "SDXL ControlNet", + "description": "Select a compatible SDXL ControlNet and supply a preprocessed control image. Input image and mask are optional for image-to-image or inpainting.", + "model_url": "https://huggingface.co/diffusers/controlnet-canny-sdxl-1.0", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "control_net", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "scheduler": "discrete", + "sampling_method": "euler_a", + "cfg_scale": 5, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "cache_unet", + "clip_skip", + "shifted_timestep" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/sdxl-inpainting.json b/examples/server/recipes/sdxl-inpainting.json new file mode 100644 index 000000000..574423d08 --- /dev/null +++ b/examples/server/recipes/sdxl-inpainting.json @@ -0,0 +1,39 @@ +{ + "schema_version": 1, + "id": "sdxl-inpainting", + "name": "SDXL Inpainting", + "description": "Select an inpainting checkpoint, then supply Init Image and Mask Image.", + "model_url": "https://huggingface.co/diffusers/stable-diffusion-xl-1.0-inpainting-0.1", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "vae", + "label": "VAE override", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "scheduler": "discrete", + "sampling_method": "euler_a", + "cfg_scale": 8, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "cache_unet", + "clip_skip", + "shifted_timestep", + "img_cfg" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/sdxl-ip-adapter.json b/examples/server/recipes/sdxl-ip-adapter.json new file mode 100644 index 000000000..b649fb1d5 --- /dev/null +++ b/examples/server/recipes/sdxl-ip-adapter.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "id": "sdxl-ip-adapter", + "name": "SDXL IP-Adapter / Plus", + "description": "Select matching SDXL ViT-H IP-Adapter or Plus weights and CLIP Vision H, then supply an IP-Adapter image.", + "model_url": "https://huggingface.co/h94/IP-Adapter", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "clip_vision", + "required": true, + "label": "CLIP Vision H" + }, + { + "id": "ip_adapter", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "scheduler": "discrete", + "sampling_method": "dpm++2m", + "cfg_scale": 5, + "steps": 20, + "width": 1024, + "height": 1024 + }, + "features": [ + "cache_unet", + "clip_skip", + "shifted_timestep" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/sdxl-photomaker.json b/examples/server/recipes/sdxl-photomaker.json new file mode 100644 index 000000000..025fb7668 --- /dev/null +++ b/examples/server/recipes/sdxl-photomaker.json @@ -0,0 +1,37 @@ +{ + "schema_version": 1, + "id": "sdxl-photomaker", + "name": "SDXL PhotoMaker", + "description": "Select PhotoMaker v1 and supply identity images. Include the img trigger in the prompt.", + "model_url": "https://huggingface.co/TencentARC/PhotoMaker", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "photo_maker", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "width": 1024, + "height": 1024 + }, + "features": [ + "cache_unet", + "clip_skip", + "shifted_timestep" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/sdxl-turbo.json b/examples/server/recipes/sdxl-turbo.json new file mode 100644 index 000000000..8650e6c58 --- /dev/null +++ b/examples/server/recipes/sdxl-turbo.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "id": "sdxl-turbo", + "name": "SDXL Turbo", + "model_url": "https://huggingface.co/stabilityai/sdxl-turbo", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "defaults": { + "cfg_scale": 1, + "steps": 1, + "width": 512, + "height": 512 + }, + "features": [ + "cache_unet", + "clip_skip", + "shifted_timestep" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/sdxs.json b/examples/server/recipes/sdxs.json new file mode 100644 index 000000000..19e218ac5 --- /dev/null +++ b/examples/server/recipes/sdxs.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "id": "sdxs", + "name": "SDXS", + "model_url": "https://huggingface.co/IDKiro/sdxs-512-dreamshaper", + "slots": [ + { + "id": "model", + "required": true + }, + { + "id": "vae", + "required": false + } + ], + "defaults": { + "cfg_scale": 1, + "steps": 1, + "width": 512, + "height": 512 + }, + "features": [ + "cache_unet", + "clip_skip" + ], + "disabled_features": [ + "cache_dit" + ] +} diff --git a/examples/server/recipes/sefi-image.json b/examples/server/recipes/sefi-image.json new file mode 100644 index 000000000..35e933360 --- /dev/null +++ b/examples/server/recipes/sefi-image.json @@ -0,0 +1,39 @@ +{ + "schema_version": 1, + "id": "sefi-image", + "name": "SeFi Image 5B RL", + "model_url": "https://huggingface.co/SeFi-Image/SeFi-Image-5B-Base", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 4B Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 4, + "steps": 20, + "sampling_method": "euler" + }, + "features": [ + "slg" + ] +} diff --git a/examples/server/recipes/sensenova-u15.json b/examples/server/recipes/sensenova-u15.json new file mode 100644 index 000000000..afd20466b --- /dev/null +++ b/examples/server/recipes/sensenova-u15.json @@ -0,0 +1,24 @@ +{ + "schema_version": 1, + "id": "sensenova-u15", + "name": "SenseNova U1.5", + "description": "Bundled MoT checkpoint.", + "model_url": "https://huggingface.co/sensenova/SenseNova-U1.5-8B-MoT", + "slots": [ + { + "id": "model", + "required": true + } + ], + "defaults": { + "flow_shift": 3.0, + "cfg_scale": 4, + "steps": 20, + "width": 2048, + "height": 2048, + "sampling_method": "euler" + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes/wan21-flf2v.json b/examples/server/recipes/wan21-flf2v.json new file mode 100644 index 000000000..02f0e80a9 --- /dev/null +++ b/examples/server/recipes/wan21-flf2v.json @@ -0,0 +1,56 @@ +{ + "schema_version": 1, + "id": "wan21-flf2v", + "name": "Wan 2.1 FLF2V 14B", + "description": "Select FLF2V weights and supply both start and end images.", + "model_url": "https://huggingface.co/Wan-AI/Wan2.1-FLF2V-14B-720P", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "t5xxl", + "required": true, + "label": "UMT5-XXL" + }, + { + "id": "clip_vision", + "required": true, + "label": "CLIP Vision H" + }, + { + "id": "vae", + "required": false, + "label": "Wan 2.1 VAE" + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "flow_shift": 16, + "video_frames": 21, + "cfg_scale": 5, + "steps": 20, + "width": 1280, + "height": 720 + }, + "features": [ + "end_image", + "flow_shift", + "init_image" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/wan21-i2v.json b/examples/server/recipes/wan21-i2v.json new file mode 100644 index 000000000..d8d8d13ee --- /dev/null +++ b/examples/server/recipes/wan21-i2v.json @@ -0,0 +1,56 @@ +{ + "schema_version": 1, + "id": "wan21-i2v", + "name": "Wan 2.1 I2V", + "description": "Image-to-video with Wan 2.1, compatible SkyReels V2 weights, or a single Wan 2.2 I2V expert. CLIP Vision is required by Wan 2.1 I2V and is unused by Wan 2.2. Defaults target Wan 2.1 480p.", + "model_url": "https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-480P", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "t5xxl", + "label": "UMT5-XXL", + "required": true + }, + { + "id": "vae", + "label": "Wan 2.1 VAE", + "required": false + }, + { + "id": "tae", + "required": false + }, + { + "id": "clip_vision", + "label": "CLIP Vision H (Wan 2.1 I2V)", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "video_frames": 33, + "fps": 16, + "flow_shift": 3, + "cfg_scale": 5, + "steps": 20, + "width": 832, + "height": 480 + }, + "features": [ + "flow_shift", + "init_image" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/wan21-t2v.json b/examples/server/recipes/wan21-t2v.json new file mode 100644 index 000000000..367611c41 --- /dev/null +++ b/examples/server/recipes/wan21-t2v.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "id": "wan21-t2v", + "name": "Wan 2.1 T2V", + "description": "Text-to-video with Wan 2.1 or a compatible single Wan 2.2 T2V expert. Defaults target Wan 2.1 14B; adjust settings for the selected weights and acceleration LoRAs.", + "model_url": "https://huggingface.co/Wan-AI/Wan2.1-T2V-14B", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "t5xxl", + "label": "UMT5-XXL", + "required": true + }, + { + "id": "vae", + "label": "Wan 2.1 VAE", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "video_frames": 33, + "fps": 16, + "flow_shift": 5, + "cfg_scale": 5, + "steps": 20, + "width": 1280, + "height": 720 + }, + "features": [ + "flow_shift" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/wan21-vace.json b/examples/server/recipes/wan21-vace.json new file mode 100644 index 000000000..7b7f7e41b --- /dev/null +++ b/examples/server/recipes/wan21-vace.json @@ -0,0 +1,52 @@ +{ + "schema_version": 1, + "id": "wan21-vace", + "name": "Wan 2.1 VACE 1.3B / 14B", + "description": "Select matching VACE weights. Supports text, reference images and control video through Video Generation.", + "model_url": "https://huggingface.co/Wan-AI/Wan2.1-VACE-14B", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "t5xxl", + "required": true, + "label": "UMT5-XXL" + }, + { + "id": "vae", + "required": false, + "label": "Wan 2.1 VAE" + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "video_frames": 33, + "cfg_scale": 5, + "steps": 20, + "width": 832, + "height": 480, + "flow_shift": 16 + }, + "features": [ + "control_frames", + "flow_shift", + "init_image", + "vace" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/wan22-i2v.json b/examples/server/recipes/wan22-i2v.json new file mode 100644 index 000000000..62afdfce4 --- /dev/null +++ b/examples/server/recipes/wan22-i2v.json @@ -0,0 +1,62 @@ +{ + "schema_version": 1, + "id": "wan22-i2v", + "name": "Wan 2.2 I2V A14B", + "description": "Image-to-video. Select a start image in Video Generation after loading. LightX2V acceleration LoRAs must match the I2V low- and high-noise models; configure LoRAs and sampling separately.", + "model_url": "https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B", + "slots": [ + { + "id": "diffusion_model", + "label": "Low-noise diffusion model", + "required": true + }, + { + "id": "high_noise_diffusion_model", + "required": true + }, + { + "id": "t5xxl", + "label": "UMT5-XXL", + "required": true + }, + { + "id": "vae", + "label": "Wan 2.1 VAE", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "video_frames": 33, + "fps": 16, + "flow_shift": 5, + "high_noise_steps": -1, + "high_noise_cfg_scale": 3.5, + "high_noise_sampling_method": "euler", + "high_noise_flow_shift": 5, + "cfg_scale": 3.5, + "steps": 20, + "width": 1280, + "height": 720, + "moe_boundary": 0.9 + }, + "features": [ + "end_image", + "flow_shift", + "init_image" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/wan22-s2v.json b/examples/server/recipes/wan22-s2v.json new file mode 100644 index 000000000..ba1ee4f7c --- /dev/null +++ b/examples/server/recipes/wan22-s2v.json @@ -0,0 +1,53 @@ +{ + "schema_version": 1, + "id": "wan22-s2v", + "name": "Wan 2.2 S2V 14B", + "description": "Supply a start image and reference audio through Video Generation.", + "model_url": "https://huggingface.co/Wan-AI/Wan2.2-S2V-14B", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "t5xxl", + "required": true, + "label": "UMT5-XXL" + }, + { + "id": "audio_encoder", + "required": true, + "label": "Wav2Vec2 Large English" + }, + { + "id": "vae", + "required": false, + "label": "Wan 2.1 VAE" + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "flow_shift": 3, + "video_frames": 33, + "cfg_scale": 4.5, + "steps": 20 + }, + "features": [ + "flow_shift", + "init_image" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/wan22-t2v.json b/examples/server/recipes/wan22-t2v.json new file mode 100644 index 000000000..905bcfa4f --- /dev/null +++ b/examples/server/recipes/wan22-t2v.json @@ -0,0 +1,60 @@ +{ + "schema_version": 1, + "id": "wan22-t2v", + "name": "Wan 2.2 T2V A14B", + "description": "Text-to-video. Use Video Generation after loading. LightX2V acceleration LoRAs must match the T2V low- and high-noise models; configure LoRAs and sampling separately.", + "model_url": "https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B", + "slots": [ + { + "id": "diffusion_model", + "label": "Low-noise diffusion model", + "required": true + }, + { + "id": "high_noise_diffusion_model", + "required": true + }, + { + "id": "t5xxl", + "label": "UMT5-XXL", + "required": true + }, + { + "id": "vae", + "label": "Wan 2.1 VAE", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "video_frames": 33, + "fps": 16, + "flow_shift": 12, + "high_noise_steps": -1, + "high_noise_cfg_scale": 4, + "high_noise_sampling_method": "euler", + "high_noise_flow_shift": 12, + "cfg_scale": 3, + "steps": 20, + "width": 1280, + "height": 720, + "moe_boundary": 0.875 + }, + "features": [ + "flow_shift" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/wan22-ti2v-5b.json b/examples/server/recipes/wan22-ti2v-5b.json new file mode 100644 index 000000000..3e2aa35b8 --- /dev/null +++ b/examples/server/recipes/wan22-ti2v-5b.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "id": "wan22-ti2v-5b", + "name": "Wan 2.2 TI2V 5B", + "description": "Use Wan 2.2 VAE. Supports text, start-image and start/end-image conditioning.", + "model_url": "https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "t5xxl", + "required": true, + "label": "UMT5-XXL" + }, + { + "id": "vae", + "required": false, + "label": "Wan 2.2 VAE" + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "video_frames": 33, + "cfg_scale": 5, + "steps": 20, + "width": 1280, + "height": 704, + "flow_shift": 5 + }, + "features": [ + "flow_shift", + "init_image" + ], + "disabled_features": [ + "img2img", + "mask_image" + ] +} diff --git a/examples/server/recipes/z-image-l2p.json b/examples/server/recipes/z-image-l2p.json new file mode 100644 index 000000000..0375f1387 --- /dev/null +++ b/examples/server/recipes/z-image-l2p.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "id": "z-image-l2p", + "name": "Z-Image L2P", + "description": "Pixel-space L2P checkpoint with Qwen 3 4B; no VAE is needed. Use dimensions divisible by 16.", + "model_url": "https://huggingface.co/zhen-nan/L2P", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 4B Text Encoder" + } + ], + "defaults": { + "sampling_method": "euler", + "scheduler": "discrete" + }, + "features": [ + "flow_shift" + ], + "disabled_features": [ + "vae_tiling", + "preview" + ] +} diff --git a/examples/server/recipes/z-image-turbo.json b/examples/server/recipes/z-image-turbo.json new file mode 100644 index 000000000..99737674e --- /dev/null +++ b/examples/server/recipes/z-image-turbo.json @@ -0,0 +1,41 @@ +{ + "schema_version": 1, + "id": "z-image-turbo", + "name": "Z-Image Turbo", + "model_url": "https://huggingface.co/Tongyi-MAI/Z-Image-Turbo", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 1, + "steps": 9, + "width": 1024, + "height": 1024 + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes/z-image.json b/examples/server/recipes/z-image.json new file mode 100644 index 000000000..bcf2da7cd --- /dev/null +++ b/examples/server/recipes/z-image.json @@ -0,0 +1,41 @@ +{ + "schema_version": 1, + "id": "z-image", + "name": "Z-Image", + "model_url": "https://huggingface.co/Tongyi-MAI/Z-Image", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 Text Encoder" + }, + { + "id": "vae", + "required": false + }, + { + "id": "tae", + "required": false + } + ], + "required_any": [ + [ + "vae", + "tae" + ] + ], + "defaults": { + "cfg_scale": 4, + "steps": 20, + "width": 720, + "height": 1280, + "sampling_method": "euler" + }, + "features": [ + "flow_shift" + ] +} diff --git a/examples/server/recipes_builtin.h.in b/examples/server/recipes_builtin.h.in new file mode 100644 index 000000000..ff31b4d2c --- /dev/null +++ b/examples/server/recipes_builtin.h.in @@ -0,0 +1,7 @@ +#ifndef __SD_EXAMPLES_SERVER_RECIPES_BUILTIN_H__ +#define __SD_EXAMPLES_SERVER_RECIPES_BUILTIN_H__ + +static const char* const k_builtin_recipes[] = { +@SD_BUILTIN_RECIPES@}; + +#endif diff --git a/examples/server/routes.h b/examples/server/routes.h index 1a4efb5b7..f77939184 100644 --- a/examples/server/routes.h +++ b/examples/server/routes.h @@ -9,3 +9,18 @@ void register_index_endpoints(httplib::Server& svr, const SDSvrParams& svr_param void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt); void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt); void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt); +void register_model_endpoints(httplib::Server& svr, ServerRuntime& rt); + +class RuntimeRoutes { + httplib::Server& server_; + ServerRuntime& runtime_; + + httplib::Server::Handler protect(httplib::Server::Handler handler); + +public: + RuntimeRoutes(httplib::Server& server, ServerRuntime& runtime) + : server_(server), runtime_(runtime) {} + + void get(const std::string& pattern, httplib::Server::Handler handler); + void post(const std::string& pattern, httplib::Server::Handler handler); +}; diff --git a/examples/server/routes_index.cpp b/examples/server/routes_index.cpp index 1341ff84a..14ad814b4 100644 --- a/examples/server/routes_index.cpp +++ b/examples/server/routes_index.cpp @@ -1,5 +1,6 @@ #include "routes.h" +#include #include #include @@ -7,7 +8,7 @@ void register_index_endpoints(httplib::Server& svr, const SDSvrParams& svr_param const std::string serve_html_path = svr_params.serve_html_path; svr.Get("/", [serve_html_path, index_html](const httplib::Request&, httplib::Response& res) { if (!serve_html_path.empty()) { - std::ifstream file(serve_html_path); + std::ifstream file(std::filesystem::u8path(serve_html_path)); if (file) { std::string content((std::istreambuf_iterator(file)), std::istreambuf_iterator()); res.set_content(content, "text/html"); diff --git a/examples/server/routes_models.cpp b/examples/server/routes_models.cpp new file mode 100644 index 000000000..ea14eb821 --- /dev/null +++ b/examples/server/routes_models.cpp @@ -0,0 +1,150 @@ +#include "routes.h" + +#include "async_jobs.h" +#include "model_catalog.h" + +static void model_error(httplib::Response& res, int status, const std::string& code, const std::string& message) { + res.status = status; + res.set_content(json({{"error", message}, {"code", code}}).dump(), "application/json"); +} + +httplib::Server::Handler RuntimeRoutes::protect(httplib::Server::Handler handler) { + ServerRuntime* runtime = &runtime_; + return [runtime, handler](const httplib::Request& req, httplib::Response& res) { + std::shared_lock lock(runtime->models->runtime_mutex, std::try_to_lock); + if (!lock.owns_lock()) { + model_error(res, 503, "model_loading", "Model is loading or unloading"); + return; + } + const bool generation = req.path == "/sdcpp/v1/img_gen" || req.path == "/sdcpp/v1/vid_gen" || + req.path == "/sdapi/v1/txt2img" || req.path == "/sdapi/v1/img2img" || + req.path == "/v1/images/generations" || req.path == "/v1/images/edits"; + if (generation && !runtime->sd_ctx) { + model_error(res, 503, "model_not_loaded", "No model is loaded"); + return; + } + handler(req, res); + }; +} + +void RuntimeRoutes::get(const std::string& pattern, httplib::Server::Handler handler) { + // Job polling and cancellation only access the job manager, even during a model transition. + server_.Get(pattern, pattern.find("/sdcpp/v1/jobs/") != std::string::npos ? handler : protect(handler)); +} + +void RuntimeRoutes::post(const std::string& pattern, httplib::Server::Handler handler) { + server_.Post(pattern, pattern.find("/sdcpp/v1/jobs/") != std::string::npos ? handler : protect(handler)); +} + +static void set_model_status(ServerRuntime& runtime, const char* state, const json& recipe, const json& selection, const json& error = nullptr) { + std::lock_guard lock(runtime.models->status_mutex); + auto& models = *runtime.models; + const uint64_t revision = models.status.value("revision", uint64_t(0)) + 1; + json elapsed = nullptr; + if (!recipe.is_null()) { + const auto now = std::chrono::steady_clock::now(); + if (std::string(state) == "loading") { + models.load_started_at = now; + elapsed = 0.0; + } else { + elapsed = std::chrono::duration(now - models.load_started_at).count(); + } + } + models.status = {{"state", state}, {"recipe", recipe}, {"models", selection}, {"error", error}, {"revision", revision}, {"load_elapsed_seconds", elapsed}}; +} + +static void change_model(ServerRuntime& runtime, const httplib::Request& req, httplib::Response& res, bool unload) { + auto& models = *runtime.models; + std::unique_lock lock(models.runtime_mutex, std::try_to_lock); + if (!lock.owns_lock()) { + model_error(res, 409, "model_busy", "Model is in use or another model operation is running"); + return; + } + { + // Admission holds runtime_mutex through enqueue. Jobs stay pending until + // generation and metadata encoding finish, so no worker can retain this context here. + std::lock_guard jobs_lock(runtime.async_job_manager->mutex); + if (count_pending_jobs(*runtime.async_job_manager) != 0) { + model_error(res, 409, "model_busy", "Finish or cancel pending generation jobs before switching models"); + return; + } + } + + json body; + json loaded_recipe; + SDContextParams context; + SDGenerationParams defaults; + if (!unload) { + try { + body = json::parse(req.body); + prepare_model_load(runtime, body, context, defaults, &loaded_recipe); + } catch (const std::exception& e) { + model_error(res, 400, "invalid_model_config", e.what()); + return; + } + } + + const json recipe = unload ? json(nullptr) : body.at("recipe"); + const json selection = unload ? json::object() : body.at("models"); + set_model_status(runtime, "loading", recipe, selection); + try { + runtime.sd_ctx = nullptr; + models.context.reset(); + models.active_recipe = nullptr; + *runtime.ctx_params = unload ? models.base_context : std::move(context); + *runtime.default_gen_params = unload ? models.base_defaults : std::move(defaults); + if (!unload) { + auto params = runtime.ctx_params->to_sd_ctx_params_t(false); + models.context.reset(new_sd_ctx(¶ms)); + if (!models.context) { + throw std::runtime_error("Model initialization failed; check the server log for component compatibility and memory errors"); + } + runtime.sd_ctx = models.context.get(); + models.active_recipe = std::move(loaded_recipe); + } + set_model_status(runtime, unload ? "empty" : "ready", recipe, selection); + res.set_content(model_status(runtime).dump(), "application/json"); + } catch (const std::exception& e) { + runtime.sd_ctx = nullptr; + models.context.reset(); + models.active_recipe = nullptr; + *runtime.ctx_params = models.base_context; + *runtime.default_gen_params = models.base_defaults; + set_model_status(runtime, "error", recipe, selection, e.what()); + model_error(res, 422, "model_load_failed", e.what()); + } +} + +void register_model_endpoints(httplib::Server& svr, ServerRuntime& runtime) { + svr.Get("/sdcpp/v1/recipes", [&runtime](const httplib::Request&, httplib::Response& res) { + auto& models = *runtime.models; + std::lock_guard lock(models.catalog_mutex); + json items = json::array(); + for (const auto& entry : models.recipes) + items.push_back(entry.second); + res.set_content(json({{"recipes", items}, {"errors", models.catalog_errors}}).dump(), "application/json"); + }); + svr.Get("/sdcpp/v1/model-files", [&runtime](const httplib::Request&, httplib::Response& res) { + auto& models = *runtime.models; + std::lock_guard lock(models.catalog_mutex); + json items = json::array(); + for (const auto& entry : models.files) { + items.push_back({{"id", entry.first}, {"name", entry.second.name}, {"category", entry.second.category}, {"root", entry.second.root.id}, {"relative_path", entry.second.relative_path}}); + } + res.set_content(json({{"files", items}, {"errors", models.catalog_errors}}).dump(), "application/json"); + }); + svr.Post("/sdcpp/v1/models/refresh", [&runtime](const httplib::Request&, httplib::Response& res) { + refresh_model_catalog(runtime); + std::lock_guard lock(runtime.models->catalog_mutex); + res.set_content(json({{"errors", runtime.models->catalog_errors}}).dump(), "application/json"); + }); + svr.Get("/sdcpp/v1/models/status", [&runtime](const httplib::Request&, httplib::Response& res) { + res.set_content(model_status(runtime).dump(), "application/json"); + }); + svr.Post("/sdcpp/v1/models/load", [&runtime](const httplib::Request& req, httplib::Response& res) { + change_model(runtime, req, res, false); + }); + svr.Post("/sdcpp/v1/models/unload", [&runtime](const httplib::Request& req, httplib::Response& res) { + change_model(runtime, req, res, true); + }); +} diff --git a/examples/server/routes_openai.cpp b/examples/server/routes_openai.cpp index af0cdfaa1..29dfe313a 100644 --- a/examples/server/routes_openai.cpp +++ b/examples/server/routes_openai.cpp @@ -7,6 +7,7 @@ #include "common/common.h" #include "common/media_io.h" #include "common/resource_owners.hpp" +#include "openai_edit_policy.h" static std::string extract_and_remove_sd_cpp_extra_args(std::string& text) { std::regex re("(.*?)"); @@ -36,7 +37,7 @@ static bool build_openai_generation_request(const httplib::Request& req, std::string output_format = j.value("output_format", "png"); int output_compression = j.value("output_compression", 100); int width = runtime.default_gen_params->width > 0 ? runtime.default_gen_params->width : 512; - int height = runtime.default_gen_params->width > 0 ? runtime.default_gen_params->height : 512; + int height = runtime.default_gen_params->height > 0 ? runtime.default_gen_params->height : 512; if (!size.empty()) { auto pos = size.find('x'); if (pos != std::string::npos) { @@ -157,7 +158,24 @@ static bool build_openai_edit_request(const httplib::Request& req, request.gen_params.height = height; request.gen_params.batch_count = n; - std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt); + std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt); + const json extra_args = sd_cpp_extra_args_str.empty() + ? json::object() + : json::parse(sd_cpp_extra_args_str, nullptr, false); + const bool has_extra_init_image_override = extra_args.is_object() && extra_args.contains("init_image"); + const bool has_extra_ref_images_override = extra_args.is_object() && extra_args.contains("ref_images"); + const bool has_extra_strength_override = extra_args.is_object() && + extra_args.contains("strength") && + extra_args["strength"].is_number(); + const bool has_extra_mask_image = extra_args.is_object() && + extra_args.contains("mask_image") && + !extra_args["mask_image"].is_null(); + const bool supports_ref_image_conditioning = sd_ctx_supports_ref_image_conditioning(runtime.sd_ctx); + const bool may_need_uploaded_init_image = !has_extra_init_image_override && + (!supports_ref_image_conditioning || !mask_bytes.empty() || + has_extra_mask_image || has_extra_strength_override || has_extra_ref_images_override); + SDImageOwner uploaded_image_for_init; + bool uploaded_image_was_first_reference = false; for (auto& bytes : images_bytes) { int img_w = 0; int img_h = 0; @@ -177,10 +195,9 @@ static bool build_openai_edit_request(const httplib::Request& req, request.gen_params.set_width_and_height_if_unset(image_owner.get().width, image_owner.get().height); if (is_first_ref_image) { - request.gen_params.init_image = image_owner; - if (request.gen_params.init_image.get().data == nullptr) { - error_message = "could not allocate init image"; - return false; + uploaded_image_was_first_reference = true; + if (may_need_uploaded_init_image) { + uploaded_image_for_init = image_owner; } } @@ -214,6 +231,21 @@ static bool build_openai_edit_request(const httplib::Request& req, return false; } + const auto init_image_action = apply_openai_edit_init_image_policy( + request.gen_params.init_image, + uploaded_image_for_init, + uploaded_image_was_first_reference, + supports_ref_image_conditioning, + !request.gen_params.ref_images.empty(), + request.gen_params.mask_image.get().data != nullptr, + has_extra_init_image_override, + has_extra_strength_override); + if (init_image_action == OpenAIEditInitImageAction::UseUploaded && + request.gen_params.init_image.get().data == nullptr) { + error_message = "could not allocate init image"; + return false; + } + // Intentionally disable prompt-embedded LoRA tag parsing for server APIs. if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) { error_message = "invalid params"; @@ -249,15 +281,18 @@ static bool execute_sync_img_gen_request(ServerRuntime& runtime, void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { ServerRuntime* runtime = &rt; + RuntimeRoutes routes(svr, rt); - svr.Get("/v1/models", [runtime](const httplib::Request&, httplib::Response& res) { + routes.get("/v1/models", [runtime](const httplib::Request&, httplib::Response& res) { json r; r["data"] = json::array(); - r["data"].push_back({{"id", "sd-cpp-local"}, {"object", "model"}, {"owned_by", "local"}}); + if (runtime->sd_ctx) { + r["data"].push_back({{"id", "sd-cpp-local"}, {"object", "model"}, {"owned_by", "local"}}); + } res.set_content(r.dump(), "application/json"); }); - svr.Post("/v1/images/generations", [runtime](const httplib::Request& req, httplib::Response& res) { + routes.post("/v1/images/generations", [runtime](const httplib::Request& req, httplib::Response& res) { try { if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) { res.status = 400; @@ -273,7 +308,7 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { return; } - LOG_VERBOSE("%s\n", request.gen_params.to_string().c_str()); + LOG_DEBUG("%s\n", request.gen_params.to_string().c_str()); SDImageVec results; if (!execute_sync_img_gen_request(*runtime, request, results, error_message)) { @@ -331,7 +366,7 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { } }); - svr.Post("/v1/images/edits", [runtime](const httplib::Request& req, httplib::Response& res) { + routes.post("/v1/images/edits", [runtime](const httplib::Request& req, httplib::Response& res) { try { if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) { res.status = 400; @@ -347,7 +382,7 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { return; } - LOG_VERBOSE("%s\n", request.gen_params.to_string().c_str()); + LOG_DEBUG("%s\n", request.gen_params.to_string().c_str()); SDImageVec results; if (!execute_sync_img_gen_request(*runtime, request, results, error_message)) { diff --git a/examples/server/routes_sdapi.cpp b/examples/server/routes_sdapi.cpp index 22a710b7e..4b8e7a5b1 100644 --- a/examples/server/routes_sdapi.cpp +++ b/examples/server/routes_sdapi.cpp @@ -10,6 +10,7 @@ #include "common/common.h" #include "common/media_io.h" #include "common/resource_owners.hpp" +#include "model_catalog.h" namespace fs = std::filesystem; @@ -28,10 +29,10 @@ static std::string extract_and_remove_sd_cpp_extra_args(std::string& text) { static fs::path resolve_display_model_path(const ServerRuntime& runtime) { const auto& ctx = *runtime.ctx_params; if (!ctx.model_path.empty()) { - return fs::path(ctx.model_path); + return fs::u8path(ctx.model_path); } if (!ctx.diffusion_model_path.empty()) { - return fs::path(ctx.diffusion_model_path); + return fs::u8path(ctx.diffusion_model_path); } return {}; } @@ -115,8 +116,9 @@ static bool build_sdapi_img_gen_request(const json& j, request.gen_params.sample_params.sample_steps = steps; request.gen_params.batch_count = batch_size; request.gen_params.sample_params.guidance.txt_cfg = cfg_scale; - request.gen_params.width = j.value("width", -1); - request.gen_params.height = j.value("height", -1); + const bool recipe_dimensions = !img2img && model_status(runtime)["recipe"].is_string(); + request.gen_params.width = j.value("width", recipe_dimensions ? runtime.default_gen_params->width : -1); + request.gen_params.height = j.value("height", recipe_dimensions ? runtime.default_gen_params->height : -1); if (!img2img && j.value("enable_hr", false)) { request.gen_params.hires_enabled = true; @@ -151,6 +153,7 @@ static bool build_sdapi_img_gen_request(const json& j, } if (j.contains("lora") && j["lora"].is_array()) { + refresh_lora_cache(runtime); request.gen_params.lora_map.clear(); request.gen_params.high_noise_lora_map.clear(); @@ -373,6 +376,7 @@ struct SDAPIPreviewGuard { void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { ServerRuntime* runtime = &rt; + RuntimeRoutes routes(svr, rt); auto sdapi_any2img = [runtime](const httplib::Request& req, httplib::Response& res, bool img2img) { try { @@ -398,7 +402,7 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { return; } - LOG_VERBOSE("%s\n", request.gen_params.to_string().c_str()); + LOG_DEBUG("%s\n", request.gen_params.to_string().c_str()); sd_img_gen_params_t img_gen_params = request.to_sd_img_gen_params_t(); SDImageVec results; @@ -473,15 +477,15 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { } }; - svr.Post("/sdapi/v1/txt2img", [sdapi_any2img](const httplib::Request& req, httplib::Response& res) { + routes.post("/sdapi/v1/txt2img", [sdapi_any2img](const httplib::Request& req, httplib::Response& res) { sdapi_any2img(req, res, false); }); - svr.Post("/sdapi/v1/img2img", [sdapi_any2img](const httplib::Request& req, httplib::Response& res) { + routes.post("/sdapi/v1/img2img", [sdapi_any2img](const httplib::Request& req, httplib::Response& res) { sdapi_any2img(req, res, true); }); - svr.Get("/sdapi/v1/loras", [runtime](const httplib::Request&, httplib::Response& res) { + routes.get("/sdapi/v1/loras", [runtime](const httplib::Request&, httplib::Response& res) { refresh_lora_cache(*runtime); json result = json::array(); @@ -489,8 +493,9 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { std::lock_guard lock(*runtime->lora_mutex); for (const auto& e : *runtime->lora_cache) { json item; - item["name"] = e.name; - item["path"] = e.path; + item["name"] = e.name; + item["path"] = e.path; + item["display_path"] = e.display_path; result.push_back(item); } } @@ -498,7 +503,7 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { res.set_content(result.dump(), "application/json"); }); - svr.Get("/sdapi/v1/upscalers", [runtime](const httplib::Request&, httplib::Response& res) { + routes.get("/sdapi/v1/upscalers", [runtime](const httplib::Request&, httplib::Response& res) { refresh_upscaler_cache(*runtime); auto make_builtin = [](const char* name) { @@ -532,7 +537,7 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { res.set_content(result.dump(), "application/json"); }); - svr.Get("/sdapi/v1/latent-upscale-modes", [](const httplib::Request&, httplib::Response& res) { + routes.get("/sdapi/v1/latent-upscale-modes", [](const httplib::Request&, httplib::Response& res) { json result = json::array({ {{"name", "Latent"}}, {{"name", "Latent (nearest)"}}, @@ -544,7 +549,7 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { res.set_content(result.dump(), "application/json"); }); - svr.Get("/sdapi/v1/samplers", [runtime](const httplib::Request&, httplib::Response& res) { + routes.get("/sdapi/v1/samplers", [runtime](const httplib::Request&, httplib::Response& res) { std::vector sampler_names; sampler_names.push_back("default"); for (int i = 0; i < SAMPLE_METHOD_COUNT; i++) { @@ -561,7 +566,7 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { res.set_content(r.dump(), "application/json"); }); - svr.Get("/sdapi/v1/schedulers", [runtime](const httplib::Request&, httplib::Response& res) { + routes.get("/sdapi/v1/schedulers", [runtime](const httplib::Request&, httplib::Response& res) { std::vector scheduler_names; scheduler_names.push_back("default"); for (int i = 0; i < SCHEDULER_COUNT; i++) { @@ -580,29 +585,31 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { res.set_content(r.dump(), "application/json"); }); - svr.Get("/sdapi/v1/sd-models", [runtime](const httplib::Request&, httplib::Response& res) { + routes.get("/sdapi/v1/sd-models", [runtime](const httplib::Request&, httplib::Response& res) { fs::path model_path = resolve_display_model_path(*runtime); json entry; - entry["title"] = model_path.stem(); - entry["model_name"] = model_path.stem(); - entry["filename"] = model_path.filename(); + entry["title"] = model_path.stem().u8string(); + entry["model_name"] = model_path.stem().u8string(); + entry["filename"] = model_path.filename().u8string(); entry["hash"] = "8888888888"; entry["sha256"] = "8888888888888888888888888888888888888888888888888888888888888888"; entry["config"] = nullptr; json r = json::array(); - r.push_back(entry); + if (runtime->sd_ctx) { + r.push_back(entry); + } res.set_content(r.dump(), "application/json"); }); - svr.Get("/sdapi/v1/options", [runtime](const httplib::Request&, httplib::Response& res) { + routes.get("/sdapi/v1/options", [runtime](const httplib::Request&, httplib::Response& res) { fs::path model_path = resolve_display_model_path(*runtime); json r; r["samples_format"] = "png"; - r["sd_model_checkpoint"] = model_path.stem(); + r["sd_model_checkpoint"] = model_path.stem().u8string(); res.set_content(r.dump(), "application/json"); }); - svr.Get("/sdapi/v1/progress", [](const httplib::Request& req, httplib::Response& res) { + routes.get("/sdapi/v1/progress", [](const httplib::Request& req, httplib::Response& res) { bool skip_current_image = false; bool has_id_param = req.has_param("id_task"); std::string req_id_task = has_id_param ? req.get_param_value("id_task") : ""; diff --git a/examples/server/routes_sdcpp.cpp b/examples/server/routes_sdcpp.cpp index 3139682f9..7e93e5edb 100644 --- a/examples/server/routes_sdcpp.cpp +++ b/examples/server/routes_sdcpp.cpp @@ -9,6 +9,7 @@ #include "common/common.h" #include "common/media_io.h" #include "common/resource_owners.hpp" +#include "model_catalog.h" namespace fs = std::filesystem; @@ -90,20 +91,21 @@ static json make_vae_tiling_json(const sd_tiling_params_t& params) { static fs::path resolve_display_model_path(const ServerRuntime& runtime) { const auto& ctx = *runtime.ctx_params; if (!ctx.model_path.empty()) { - return fs::path(ctx.model_path); + return fs::u8path(ctx.model_path); } if (!ctx.diffusion_model_path.empty()) { - return fs::path(ctx.diffusion_model_path); + return fs::u8path(ctx.diffusion_model_path); } return {}; } -static json make_sample_params_json(const sd_sample_params_t& sample_params, const std::vector& skip_layers) { +static json make_sample_params_json(const sd_sample_params_t& sample_params, const std::vector& skip_layers, const std::vector& custom_sigmas = {}) { const auto& guidance = sample_params.guidance; return { {"scheduler", capability_scheduler_name(sample_params.scheduler)}, {"sample_method", capability_sample_method_name(sample_params.sample_method)}, {"sample_steps", sample_params.sample_steps}, + {"custom_sigmas", custom_sigmas}, {"eta", finite_number_or_null(sample_params.eta)}, {"shifted_timestep", sample_params.shifted_timestep}, {"flow_shift", finite_number_or_null(sample_params.flow_shift)}, @@ -153,7 +155,7 @@ static json make_img_gen_defaults_json(const SDGenerationParams& defaults, const {"increase_ref_index", defaults.increase_ref_index}, {"control_strength", defaults.control_strength}, {"ip_adapter_strength", defaults.ip_adapter_strength}, - {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers)}, + {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers, defaults.custom_sigmas)}, {"hires", make_hires_json(defaults)}, {"vae_tiling_params", make_vae_tiling_json(defaults.vae_tiling_params)}, {"cache_mode", defaults.cache_mode}, @@ -179,7 +181,7 @@ static json make_vid_gen_defaults_json(const SDGenerationParams& defaults, const {"fps", defaults.fps}, {"moe_boundary", defaults.moe_boundary}, {"vace_strength", defaults.vace_strength}, - {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers)}, + {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers, defaults.custom_sigmas)}, {"high_noise_sample_params", make_sample_params_json(defaults.high_noise_sample_params, defaults.high_noise_skip_layers)}, {"hires", make_hires_json(defaults)}, {"vae_tiling_params", make_vae_tiling_json(defaults.vae_tiling_params)}, @@ -266,6 +268,7 @@ static json make_capabilities_json(ServerRuntime& runtime) { available_loras.push_back({ {"name", entry.name}, {"path", entry.path}, + {"display_path", entry.display_path}, }); } } @@ -344,18 +347,34 @@ static json make_capabilities_json(ServerRuntime& runtime) { default_vid_output_format = video_output_formats[0].get(); } + const json generation_features = model_generation_features(runtime); + auto recipe_features = [&](json features, bool video) { + features["init_image"] = generation_features.at(video ? "init_image" : "img2img"); + for (const char* key : {"mask_image", "ref_images", "end_image", "control_frames", "lora", "vae_tiling", "cache", "preview"}) { + if (features.contains(key) || std::string(key) == "ref_images") + features[key] = generation_features.at(key); + } + if (video) + features["high_noise_sample_params"] = generation_features.at("high_noise"); + else { + features["mask_image"] = generation_features.at("mask_image").get() && generation_features.at("img2img").get(); + features["control_image"] = generation_features.at("controlnet").get() || generation_features.at("control_image").get(); + features["ip_adapter_image"] = generation_features.at("ip_adapter"); + } + return features; + }; json defaults_by_mode = json::object(); json output_formats_by_mode = json::object(); json features_by_mode = json::object(); if (supports_img) { defaults_by_mode["img_gen"] = make_img_gen_defaults_json(defaults, default_img_output_format); output_formats_by_mode["img_gen"] = image_output_formats; - features_by_mode["img_gen"] = make_img_gen_features_json(); + features_by_mode["img_gen"] = recipe_features(make_img_gen_features_json(), false); } if (supports_vid) { defaults_by_mode["vid_gen"] = make_vid_gen_defaults_json(defaults, default_vid_output_format); output_formats_by_mode["vid_gen"] = video_output_formats; - features_by_mode["vid_gen"] = make_vid_gen_features_json(); + features_by_mode["vid_gen"] = recipe_features(make_vid_gen_features_json(), true); } json top_level_defaults = json::object(); @@ -383,19 +402,21 @@ static json make_capabilities_json(ServerRuntime& runtime) { {"stem", model_path.stem().u8string()}, {"path", model_path.u8string()}, }; - result["current_mode"] = current_mode; - result["supported_modes"] = supported_modes; - result["defaults"] = top_level_defaults; - result["defaults_by_mode"] = defaults_by_mode; - result["limits"] = { - {"min_width", 64}, - {"max_width", 4096}, - {"min_height", 64}, - {"max_height", 4096}, - {"max_batch_count", 8}, - {"max_queue_size", manager.max_pending_jobs}, - {"max_upscale_width", k_max_upscale_dimension}, - {"max_upscale_height", k_max_upscale_dimension}, + result["generation_features"] = generation_features; + result["runtime"] = model_status(runtime); + result["current_mode"] = current_mode; + result["supported_modes"] = supported_modes; + result["defaults"] = top_level_defaults; + result["defaults_by_mode"] = defaults_by_mode; + result["limits"] = { + {"min_width", 64}, + {"max_width", 4096}, + {"min_height", 64}, + {"max_height", 4096}, + {"max_batch_count", 8}, + {"max_queue_size", manager.max_pending_jobs}, + {"max_upscale_width", k_max_upscale_dimension}, + {"max_upscale_height", k_max_upscale_dimension}, }; result["samplers"] = samplers; result["schedulers"] = schedulers; @@ -470,13 +491,14 @@ static bool parse_vid_gen_request(const json& body, void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { ServerRuntime* runtime = &rt; + RuntimeRoutes routes(svr, rt); - svr.Get("/sdcpp/v1/capabilities", [runtime](const httplib::Request&, httplib::Response& res) { + routes.get("/sdcpp/v1/capabilities", [runtime](const httplib::Request&, httplib::Response& res) { res.status = 200; res.set_content(make_capabilities_json(*runtime).dump(), "application/json"); }); - svr.Post("/sdcpp/v1/upscale", [runtime](const httplib::Request& req, httplib::Response& res) { + routes.post("/sdcpp/v1/upscale", [runtime](const httplib::Request& req, httplib::Response& res) { try { if (req.body.empty()) { res.status = 400; @@ -641,7 +663,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { } }); - svr.Post("/sdcpp/v1/img_gen", [runtime](const httplib::Request& req, httplib::Response& res) { + routes.post("/sdcpp/v1/img_gen", [runtime](const httplib::Request& req, httplib::Response& res) { try { if (req.body.empty()) { res.status = 400; @@ -667,7 +689,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { std::shared_ptr job = std::make_shared(); job->kind = AsyncJobKind::ImgGen; job->status = AsyncJobStatus::Queued; - job->created_at = unix_timestamp_now(); + job->created_at = unix_timestamp_now_ms(); job->img_gen = std::move(request); { @@ -703,7 +725,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { } }); - svr.Post("/sdcpp/v1/vid_gen", [runtime](const httplib::Request& req, httplib::Response& res) { + routes.post("/sdcpp/v1/vid_gen", [runtime](const httplib::Request& req, httplib::Response& res) { try { if (req.body.empty()) { res.status = 400; @@ -729,7 +751,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { std::shared_ptr job = std::make_shared(); job->kind = AsyncJobKind::VidGen; job->status = AsyncJobStatus::Queued; - job->created_at = unix_timestamp_now(); + job->created_at = unix_timestamp_now_ms(); job->vid_gen = std::move(request); { @@ -765,7 +787,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { } }); - svr.Get(R"(/sdcpp/v1/jobs/([A-Za-z0-9_\-]+))", [runtime](const httplib::Request& req, httplib::Response& res) { + routes.get(R"(/sdcpp/v1/jobs/([A-Za-z0-9_\-]+))", [runtime](const httplib::Request& req, httplib::Response& res) { AsyncJobManager& manager = *runtime->async_job_manager; std::lock_guard lock(manager.mutex); purge_expired_jobs(manager); @@ -787,7 +809,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { res.set_content(make_async_job_json(manager, *it->second).dump(), "application/json"); }); - svr.Post(R"(/sdcpp/v1/jobs/([A-Za-z0-9_\-]+)/cancel)", [runtime](const httplib::Request& req, httplib::Response& res) { + routes.post(R"(/sdcpp/v1/jobs/([A-Za-z0-9_\-]+)/cancel)", [runtime](const httplib::Request& req, httplib::Response& res) { AsyncJobManager& manager = *runtime->async_job_manager; std::lock_guard lock(manager.mutex); purge_expired_jobs(manager); diff --git a/examples/server/runtime.cpp b/examples/server/runtime.cpp index 1a71cf76f..34140ea35 100644 --- a/examples/server/runtime.cpp +++ b/examples/server/runtime.cpp @@ -11,6 +11,10 @@ #include "common/common.h" #include "common/log.h" +#include "model_catalog.h" + +#include +#include namespace fs = std::filesystem; @@ -22,7 +26,7 @@ static std::string lower_ascii(std::string value) { } static bool is_supported_model_ext(const fs::path& p) { - auto ext = lower_ascii(p.extension().string()); + auto ext = lower_ascii(p.extension().u8string()); return ext == ".gguf" || ext == ".pt" || ext == ".pth" || ext == ".safetensors"; } @@ -210,6 +214,13 @@ ArgOptions SDSvrParams::get_options() { options.manual_options = { {"-h", "--help", "show this help message and exit", on_help_arg}, + {"", "--model-dir", "model root; repeat to search multiple directories (default: models)", + [this](int argc, const char** argv, int index) { + if (++index >= argc || argv[index][0] == '\0') + return -1; + model_dirs.push_back(argv_to_utf8(index, argv)); + return 1; + }}, }; add_log_options(options, log_level); return options; @@ -226,7 +237,7 @@ bool SDSvrParams::validate() { return false; } - if (!serve_html_path.empty() && !fs::exists(serve_html_path)) { + if (!serve_html_path.empty() && !fs::exists(fs::u8path(serve_html_path))) { LOG_ERROR("error: serve_html_path file does not exist: %s", serve_html_path.c_str()); return false; } @@ -253,71 +264,79 @@ std::string SDSvrParams::to_string() const { void refresh_lora_cache(ServerRuntime& rt) { std::vector new_cache; - - if (rt.ctx_params->lora_model_dir.empty()) { - std::lock_guard lock(*rt.lora_mutex); - rt.lora_cache->clear(); - return; + std::vector directories; + if (!rt.ctx_params->lora_model_dir.empty()) { + directories.push_back(make_model_root(fs::u8path(rt.ctx_params->lora_model_dir), true)); + } else { + for (const auto& root : get_model_roots(*rt.svr_params)) { + try { + const auto paths = model_category_directories(root, "loras"); + for (size_t i = 0; i < paths.size(); ++i) { + if (!model_path_inside(root.path, fs::canonical(paths[i]))) { + LOG_WARN("LoRA directory escapes model directory: %s", paths[i].u8string().c_str()); + continue; + } + directories.push_back(make_model_root(paths[i], root.primary && lower_ascii(paths[i].filename().u8string()) == "loras")); + } + } catch (const std::exception& e) { + LOG_WARN("error while finding LoRA directories in %s: %s", root.label.c_str(), e.what()); + } + } } - - fs::path lora_dir = rt.ctx_params->lora_model_dir; - std::error_code ec; - if (fs::exists(lora_dir, ec) && !ec && fs::is_directory(lora_dir, ec) && !ec) { + std::set seen; + std::map names; + for (const auto& directory : directories) { try { - auto it = fs::recursive_directory_iterator( - lora_dir, - fs::directory_options::skip_permission_denied, - ec); - auto end = fs::recursive_directory_iterator(); - while (!ec && it != end) { - std::error_code entry_ec; - bool is_reg = it->is_regular_file(entry_ec); - if (entry_ec || !is_reg) { - it.increment(ec); + if (!fs::exists(directory.path)) + continue; + for (fs::recursive_directory_iterator it(directory.path), end; it != end; ++it) { + std::error_code ec; + const auto canonical = fs::canonical(it->path(), ec); + if (ec || !model_path_inside(directory.path, canonical)) { + it.disable_recursion_pending(); + LOG_WARN("skipping inaccessible LoRA or path outside LoRA directory: %s", it->path().u8string().c_str()); continue; } - const fs::path& p = it->path(); - if (!is_supported_model_ext(p)) { - it.increment(ec); + const auto ext = lower_ascii(it->path().extension().u8string()); + if (!it->is_regular_file() || (!is_supported_model_ext(it->path()) && ext != ".ckpt" && ext != ".sft") || + !seen.insert(model_path_key(canonical)).second) continue; - } - - LoraEntry lora_entry; - lora_entry.name = p.stem().u8string(); - lora_entry.fullpath = p.u8string(); - std::string rel = p.lexically_relative(lora_dir).u8string(); - std::replace(rel.begin(), rel.end(), '\\', '/'); - lora_entry.path = rel; - - new_cache.push_back(std::move(lora_entry)); - it.increment(ec); + const auto relative = it->path().lexically_relative(directory.path).generic_u8string(); + LoraEntry entry; + entry.name = it->path().stem().u8string(); + entry.path = directory.primary ? relative : directory.id + "/" + relative; + entry.fullpath = it->path().u8string(); + entry.display_path = relative; + entry.root = directory.path; + ++names[relative]; + new_cache.push_back(std::move(entry)); } } catch (const std::exception& e) { - LOG_WARN("error while scanning lora directory '%s': %s", lora_dir.string().c_str(), e.what()); - return; + LOG_WARN("error while scanning LoRA directory '%s': %s", directory.path.u8string().c_str(), e.what()); } } - - if (ec) { - LOG_WARN("error while scanning lora directory '%s': %s", lora_dir.string().c_str(), ec.message().c_str()); - return; + for (auto& entry : new_cache) { + if (names[entry.display_path] > 1) + entry.display_path = fs::u8path(entry.fullpath).generic_u8string(); } - std::sort(new_cache.begin(), new_cache.end(), [](const LoraEntry& a, const LoraEntry& b) { return a.path < b.path; }); - - { - std::lock_guard lock(*rt.lora_mutex); - *rt.lora_cache = std::move(new_cache); - } + std::lock_guard lock(*rt.lora_mutex); + *rt.lora_cache = std::move(new_cache); } std::string get_lora_full_path(ServerRuntime& rt, const std::string& path) { std::lock_guard lock(*rt.lora_mutex); auto it = std::find_if(rt.lora_cache->begin(), rt.lora_cache->end(), [&](const LoraEntry& entry) { return entry.path == path; }); - return it != rt.lora_cache->end() ? it->fullpath : ""; + if (it == rt.lora_cache->end()) + return ""; + std::error_code ec; + const auto canonical = fs::canonical(fs::u8path(it->fullpath), ec); + if (ec || !model_path_inside(it->root, canonical) || !fs::is_regular_file(canonical, ec) || ec) + return ""; + return canonical.u8string(); } void refresh_upscaler_cache(ServerRuntime& rt) { @@ -328,7 +347,7 @@ void refresh_upscaler_cache(ServerRuntime& rt) { previous_cache = *rt.upscaler_cache; } - fs::path upscaler_dir = rt.ctx_params->hires_upscalers_dir; + fs::path upscaler_dir = fs::u8path(rt.ctx_params->hires_upscalers_dir); std::error_code ec; if (fs::exists(upscaler_dir, ec) && !ec && fs::is_directory(upscaler_dir, ec) && !ec) { try { @@ -384,13 +403,13 @@ void refresh_upscaler_cache(ServerRuntime& rt) { it.increment(ec); } } catch (const std::exception& e) { - LOG_WARN("error while scanning upscalers directory '%s': %s", upscaler_dir.string().c_str(), e.what()); + LOG_WARN("error while scanning upscalers directory '%s': %s", upscaler_dir.u8string().c_str(), e.what()); return; } } if (ec) { - LOG_WARN("error while scanning upscalers directory '%s': %s", upscaler_dir.string().c_str(), ec.message().c_str()); + LOG_WARN("error while scanning upscalers directory '%s': %s", upscaler_dir.u8string().c_str(), ec.message().c_str()); return; } @@ -404,8 +423,8 @@ void refresh_upscaler_cache(ServerRuntime& rt) { } } -int64_t unix_timestamp_now() { - return std::chrono::duration_cast( +int64_t unix_timestamp_now_ms() { + return std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); } diff --git a/examples/server/runtime.h b/examples/server/runtime.h index 5baa633a6..d9272f6c2 100644 --- a/examples/server/runtime.h +++ b/examples/server/runtime.h @@ -17,14 +17,16 @@ using json = nlohmann::json; struct ArgOptions; struct SDContextParams; struct AsyncJobManager; +struct ServerModels; struct SDSvrParams { std::string listen_ip = "127.0.0.1"; int listen_port = 1234; std::string serve_html_path; - bool normal_exit = false; + std::vector model_dirs; + bool normal_exit = false; sd_log_level_t log_level = SD_LOG_INFO; - bool color = false; + bool color = false; ArgOptions get_options(); bool validate(); @@ -36,6 +38,8 @@ struct LoraEntry { std::string name; std::string path; std::string fullpath; + std::string display_path; + std::filesystem::path root; }; struct UpscalerEntry { @@ -43,7 +47,7 @@ struct UpscalerEntry { std::string path; std::string fullpath; std::string model_name; - int scale = 4; + int scale = 4; int image_upscale_factor = 0; uintmax_t file_size = 0; std::filesystem::file_time_type last_modified; @@ -53,13 +57,14 @@ struct ServerRuntime { sd_ctx_t* sd_ctx; std::mutex* sd_ctx_mutex; const SDSvrParams* svr_params; - const SDContextParams* ctx_params; - const SDGenerationParams* default_gen_params; + SDContextParams* ctx_params; + SDGenerationParams* default_gen_params; std::vector* lora_cache; std::mutex* lora_mutex; std::vector* upscaler_cache; std::mutex* upscaler_mutex; AsyncJobManager* async_job_manager; + ServerModels* models; }; struct ImgGenJobRequest { @@ -105,4 +110,4 @@ std::string unsupported_generation_mode_error(SDMode mode); void refresh_lora_cache(ServerRuntime& rt); std::string get_lora_full_path(ServerRuntime& rt, const std::string& path); void refresh_upscaler_cache(ServerRuntime& rt); -int64_t unix_timestamp_now(); +int64_t unix_timestamp_now_ms(); diff --git a/ggml b/ggml index d25b121ce..dc998771e 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit d25b121ce1d6ae0c7e62ad080460d9fc1e46dd93 +Subproject commit dc998771e700d49bbdf9a39cede22bb5b3311767 diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index 267e7a7b5..404d8cbdd 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -486,6 +486,8 @@ SD_API int32_t sd_get_num_physical_cores(); SD_API const char* sd_get_system_info(); SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx); SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx); +/** Returns whether the loaded model consumes ref_images as conditioning inputs. */ +SD_API bool sd_ctx_supports_ref_image_conditioning(const sd_ctx_t* sd_ctx); // ControlNet hot-swap APIs are not safe to call while generation is in flight. SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path); diff --git a/scripts/unsloth/ggml-patches/0001-ggml-cuda-h3-speed.patch b/scripts/unsloth/ggml-patches/0001-ggml-cuda-h3-speed.patch index 04cc94450..bac37e747 100644 --- a/scripts/unsloth/ggml-patches/0001-ggml-cuda-h3-speed.patch +++ b/scripts/unsloth/ggml-patches/0001-ggml-cuda-h3-speed.patch @@ -20,132 +20,11 @@ in one tiled H3 VAE decode. The flag is now read once at registration. src/ggml-cuda/ggml-cuda.cu | 55 ++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 8 deletions(-) -diff --git a/src/ggml-cuda/ggml-cuda.cu b/src/ggml-cuda/ggml-cuda.cu -index 5e9c323b..a153de57 100644 ---- a/src/ggml-cuda/ggml-cuda.cu -+++ b/src/ggml-cuda/ggml-cuda.cu -@@ -2431,6 +2431,25 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_q(const ggml_tensor * tensor) { - return use_mul_mat_vec_q; - } - -+// Large-batch matmul with a quantized weight: dequantize the weight to BF16 once and run a cuBLAS -+// GEMM (FP32 accumulate, FP32 output) instead of MMQ. MMQ quantizes the activations to q8_1 and runs -+// int8 MMA, which is the right call for LLM prompt sizes, but at diffusion-transformer sizes (10^4 -+// tokens per call) a tensor-core BF16 GEMM is faster on GPUs whose BF16 rate is not far below their -+// int8 rate, and it is more accurate (activations stay BF16 instead of 8-bit). -+// GGML_CUDA_QUANT_CUBLAS_MIN_BATCH=: use the BF16 path when src1 has at least this many rows; -+// 0 disables it (stock MMQ everywhere). Unset = the per-architecture default below. -+static int64_t ggml_cuda_quant_cublas_min_batch(int cc) { -+ static const int64_t env = [] { -+ const char * e = getenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH"); -+ return e != nullptr ? (int64_t) atoll(e) : (int64_t) -1; -+ }(); -+ if (env >= 0) { -+ return env; -+ } -+ GGML_UNUSED(cc); -+ return 0; -+} -+ - static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_TENSOR_BINARY_OP_LOCALS - -@@ -2496,6 +2515,23 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor - ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst); - return; - } -+ if (ggml_is_quantized(src0->type) && ne11 > MMVQ_MAX_BATCH_SIZE && GGML_CUDA_CC_IS_NVIDIA(cc) && -+ bf16_mma_hardware_available(cc) && ggml_get_to_bf16_cuda(src0->type) != nullptr) { -+ const int64_t min_batch = ggml_cuda_quant_cublas_min_batch(cc); -+ if (min_batch > 0 && ne11*ne12*ne13 >= min_batch) { -+ // GGML_CUDA_QUANT_CUBLAS_TYPE=f32 dequantizes to FP32 instead: slow, for accuracy oracles only. -+ static const bool f32_oracle = [] { -+ const char * e = getenv("GGML_CUDA_QUANT_CUBLAS_TYPE"); -+ return e != nullptr && (strcmp(e, "f32") == 0 || strcmp(e, "fp32") == 0); -+ }(); -+ if (f32_oracle) { -+ ggml_cuda_mul_mat_cublas_impl(ctx, src0, src1, dst); -+ } else { -+ ggml_cuda_mul_mat_cublas_impl(ctx, src0, src1, dst); -+ } -+ return; -+ } -+ } - if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) { - ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst); - return; -@@ -5593,6 +5629,10 @@ struct ggml_backend_cuda_device_context { - std::string description; - std::string pci_bus_id; - int op_offload_min_batch_size; -+ // cudaDeviceProp::integrated, read once at registration. cudaGetDeviceProperties costs ~1-2 ms per call, and -+ // get_memory / get_type are polled per graph by callers that budget VRAM (stable-diffusion.cpp asks thousands -+ // of times per tiled VAE decode, ~5 s of host time per H3 video on a B200 host). -+ bool integrated = false; - }; - - static const char * ggml_backend_cuda_device_get_name(ggml_backend_dev_t dev) { -@@ -5696,12 +5736,9 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * - // ref: https://github.com/ggml-org/llama.cpp/pull/17368 - #if defined(__linux__) && !defined(GGML_USE_HIP) - // Check if this is a UMA (Unified Memory Architecture) system -- cudaDeviceProp prop; -- CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device))); -- - // Check if UMA is explicitly enabled via environment variable - bool uma_env = getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") != nullptr; -- bool is_uma = prop.integrated > 0 || uma_env; -+ bool is_uma = ctx->integrated || uma_env; - - if (is_uma) { - // For UMA systems (like DGX Spark), use system memory info -@@ -5725,10 +5762,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * - static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend_dev_t dev) { - ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *) dev->context; - -- cudaDeviceProp prop; -- CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device))); -- -- return prop.integrated -+ return ctx->integrated - ? GGML_BACKEND_DEVICE_TYPE_IGPU - : GGML_BACKEND_DEVICE_TYPE_GPU; - } -@@ -6503,6 +6537,11 @@ ggml_backend_reg_t ggml_backend_cuda_reg() { - dev_ctx->device = i; - dev_ctx->name = GGML_CUDA_NAME + std::to_string(i); - dev_ctx->description = ggml_cuda_device_description(i); -+ { -+ cudaDeviceProp prop; -+ CUDA_CHECK(cudaGetDeviceProperties(&prop, physical_id)); -+ dev_ctx->integrated = prop.integrated > 0; -+ } - - char pci_bus_id[32] = {}; - CUDA_CHECK(cudaDeviceGetPCIBusId(pci_bus_id, sizeof(pci_bus_id), physical_id)); --- -2.43.0 - - -From 09f9ed593f044105ff5e206b1003327275c08b9b Mon Sep 17 00:00:00 2001 -From: Daniel Han -Date: Wed, 30 Sep 2026 22:54:01 -0700 -Subject: [PATCH 2/2] ggml-cuda: vectorized row copy for strided F32->F32/F16 - copies with contiguous rows - -cpy_scalar recomputes the 4-D source and destination index (64-bit div/mod) for -every element, so the permuted ggml_cont copies of a DiT's Q/K/V (rows of -head_dim floats) run at a few percent of memory bandwidth: a 411 MB -[128,42,19108] permute copy takes 1.67 ms on a B200, 0.30 ms as one float4 per -thread with the index math done once per row. Pure copy / same per-element -conversion, so the output is bit-identical; GGML_CUDA_CPY_ROWS=0 restores -cpy_scalar. +[Ported onto leejet/ggml dc998771 (stable-diffusion.cpp master-982): the diff below is + the whole patch as one diff, regenerated from the port.] --- - src/ggml-cuda/cpy.cu | 83 ++++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 83 insertions(+) - diff --git a/src/ggml-cuda/cpy.cu b/src/ggml-cuda/cpy.cu -index c32bb36b..4aa183af 100644 +index 5689f21b..a85d84bc 100644 --- a/src/ggml-cuda/cpy.cu +++ b/src/ggml-cuda/cpy.cu @@ -1,4 +1,5 @@ @@ -239,7 +118,7 @@ index c32bb36b..4aa183af 100644 template static void ggml_cpy_scalar_cuda( const char * cx, char * cdst, const int64_t ne, -@@ -509,6 +588,10 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg +@@ -510,6 +589,10 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg ggml_cpy_scalar_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } @@ -250,6 +129,105 @@ index c32bb36b..4aa183af 100644 } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { if (can_be_transposed) { ggml_cpy_scalar_cuda --- -2.43.0 - +diff --git a/src/ggml-cuda/ggml-cuda.cu b/src/ggml-cuda/ggml-cuda.cu +index c28589ae..8a295c19 100644 +--- a/src/ggml-cuda/ggml-cuda.cu ++++ b/src/ggml-cuda/ggml-cuda.cu +@@ -2561,6 +2561,25 @@ static bool ggml_cuda_match_shared_expert(const ggml_cgraph * graph, int routed_ + return true; + } + ++// Large-batch matmul with a quantized weight: dequantize the weight to BF16 once and run a cuBLAS ++// GEMM (FP32 accumulate, FP32 output) instead of MMQ. MMQ quantizes the activations to q8_1 and runs ++// int8 MMA, which is the right call for LLM prompt sizes, but at diffusion-transformer sizes (10^4 ++// tokens per call) a tensor-core BF16 GEMM is faster on GPUs whose BF16 rate is not far below their ++// int8 rate, and it is more accurate (activations stay BF16 instead of 8-bit). ++// GGML_CUDA_QUANT_CUBLAS_MIN_BATCH=: use the BF16 path when src1 has at least this many rows; ++// 0 disables it (stock MMQ everywhere). Unset = the per-architecture default below. ++static int64_t ggml_cuda_quant_cublas_min_batch(int cc) { ++ static const int64_t env = [] { ++ const char * e = getenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH"); ++ return e != nullptr ? (int64_t) atoll(e) : (int64_t) -1; ++ }(); ++ if (env >= 0) { ++ return env; ++ } ++ GGML_UNUSED(cc); ++ return 0; ++} ++ + static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_TENSOR_BINARY_OP_LOCALS + +@@ -2625,6 +2644,23 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor + ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst); + return; + } ++ if (ggml_is_quantized(src0->type) && ne11 > MMVQ_MAX_BATCH_SIZE && GGML_CUDA_CC_IS_NVIDIA(cc) && ++ bf16_mma_hardware_available(cc) && ggml_get_to_bf16_cuda(src0->type) != nullptr) { ++ const int64_t min_batch = ggml_cuda_quant_cublas_min_batch(cc); ++ if (min_batch > 0 && ne11*ne12*ne13 >= min_batch) { ++ // GGML_CUDA_QUANT_CUBLAS_TYPE=f32 dequantizes to FP32 instead: slow, for accuracy oracles only. ++ static const bool f32_oracle = [] { ++ const char * e = getenv("GGML_CUDA_QUANT_CUBLAS_TYPE"); ++ return e != nullptr && (strcmp(e, "f32") == 0 || strcmp(e, "fp32") == 0); ++ }(); ++ if (f32_oracle) { ++ ggml_cuda_mul_mat_cublas_impl(ctx, src0, src1, dst); ++ } else { ++ ggml_cuda_mul_mat_cublas_impl(ctx, src0, src1, dst); ++ } ++ return; ++ } ++ } + if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) { + ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst); + return; +@@ -5881,6 +5917,10 @@ struct ggml_backend_cuda_device_context { + std::string description; + std::string pci_bus_id; + int op_offload_min_batch_size; ++ // cudaDeviceProp::integrated, read once at registration. cudaGetDeviceProperties costs ~1-2 ms per call, and ++ // get_memory / get_type are polled per graph by callers that budget VRAM (stable-diffusion.cpp asks thousands ++ // of times per tiled VAE decode, ~5 s of host time per H3 video on a B200 host). ++ bool integrated = false; + }; + + static const char * ggml_backend_cuda_device_get_name(ggml_backend_dev_t dev) { +@@ -5984,11 +6024,8 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * + // ref: https://github.com/ggml-org/llama.cpp/pull/17368 + #if defined(__linux__) && !defined(GGML_USE_HIP) + // Check if this is a UMA (Unified Memory Architecture) system +- cudaDeviceProp prop; +- CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device))); +- + // Check if UMA is explicitly enabled via environment variable +- bool is_uma = prop.integrated > 0 || ggml_cuda_unified_memory_enabled(); ++ bool is_uma = ctx->integrated || ggml_cuda_unified_memory_enabled(); + + if (is_uma) { + // For UMA systems (like DGX Spark), use system memory info +@@ -6012,10 +6049,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * + static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend_dev_t dev) { + ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *) dev->context; + +- cudaDeviceProp prop; +- CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device))); +- +- return prop.integrated ++ return ctx->integrated + ? GGML_BACKEND_DEVICE_TYPE_IGPU + : GGML_BACKEND_DEVICE_TYPE_GPU; + } +@@ -6795,6 +6829,11 @@ ggml_backend_reg_t ggml_backend_cuda_reg() { + dev_ctx->device = i; + dev_ctx->name = GGML_CUDA_NAME + std::to_string(i); + dev_ctx->description = ggml_cuda_device_description(i); ++ { ++ cudaDeviceProp prop; ++ CUDA_CHECK(cudaGetDeviceProperties(&prop, physical_id)); ++ dev_ctx->integrated = prop.integrated > 0; ++ } + + char pci_bus_id[32] = {}; + CUDA_CHECK(cudaDeviceGetPCIBusId(pci_bus_id, sizeof(pci_bus_id), physical_id)); diff --git a/scripts/unsloth/ggml-patches/0002-ggml-cuda-fa-longseq.patch b/scripts/unsloth/ggml-patches/0002-ggml-cuda-fa-longseq.patch index ca9141e54..190aac491 100644 --- a/scripts/unsloth/ggml-patches/0002-ggml-cuda-fa-longseq.patch +++ b/scripts/unsloth/ggml-patches/0002-ggml-cuda-fa-longseq.patch @@ -33,8 +33,37 @@ tiles, several Q tiles and GQA broadcast. 4 files changed, 184 insertions(+), 15 deletions(-) create mode 100644 src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_128-ncols2_1.cu +[Ported onto leejet/ggml dc998771 (stable-diffusion.cpp master-982): the diff below is + the whole patch as one diff, regenerated from the port.] +--- +diff --git a/src/ggml-cuda/fattn-common.cuh b/src/ggml-cuda/fattn-common.cuh +index 41d94833..82f43137 100644 +--- a/src/ggml-cuda/fattn-common.cuh ++++ b/src/ggml-cuda/fattn-common.cuh +@@ -981,7 +981,10 @@ template + void launch_fattn( + ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, const int nwarps, const size_t nbytes_shared, + const int nbatch_fa, const bool need_f16_K, const bool need_f16_V, const bool stream_k, const bool use_sparse, +- const int warp_size = WARP_SIZE, const bool async_kv_preload = false ++ const int warp_size = WARP_SIZE, const bool async_kv_preload = false, ++ // Optional: size the grid from the occupancy of this kernel instead of fattn_kernel. A drop-in kernel variant launched ++ // with the grid of the kernel it replaces splits the work (stream-k seams) identically and so gives identical results. ++ fattn_kernel_t occupancy_kernel = nullptr, const size_t occupancy_nbytes_shared = 0 + ) { + constexpr int ncols = ncols1 * ncols2; + +@@ -1134,7 +1137,8 @@ void launch_fattn( + + const dim3 block_dim(warp_size, nwarps, 1); + int max_blocks_per_sm = 1; // Max. number of active blocks limited by occupancy. +- CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks_per_sm, fattn_kernel, block_dim.x * block_dim.y * block_dim.z, nbytes_shared)); ++ CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks_per_sm, occupancy_kernel ? occupancy_kernel : fattn_kernel, ++ block_dim.x * block_dim.y * block_dim.z, occupancy_kernel ? occupancy_nbytes_shared : nbytes_shared)); + GGML_ASSERT(max_blocks_per_sm > 0); + int parallel_blocks = max_blocks_per_sm; + diff --git a/src/ggml-cuda/fattn-mma-f16.cuh b/src/ggml-cuda/fattn-mma-f16.cuh -index 449a77c5..308218f1 100644 +index 296aa2b3..e011c8d5 100644 --- a/src/ggml-cuda/fattn-mma-f16.cuh +++ b/src/ggml-cuda/fattn-mma-f16.cuh @@ -61,6 +61,10 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co @@ -48,7 +77,7 @@ index 449a77c5..308218f1 100644 GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 8, 64, 4, 64, 96, 64, 64, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 16, 64, 4, 32, 96, 64, 64, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 32, 128, 2, 32, 96, 64, 64, 2, true); -@@ -387,6 +391,24 @@ static constexpr __device__ int ggml_cuda_fattn_mma_get_nstages( +@@ -378,6 +382,24 @@ static constexpr __device__ int ggml_cuda_fattn_mma_get_nstages( #endif // CP_ASYNC_AVAILABLE } @@ -72,8 +101,8 @@ index 449a77c5..308218f1 100644 + // ------------------------------------------------------------------------------------------------------------------ - template -@@ -581,7 +603,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask( + template +@@ -573,7 +595,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask( template= 0 ? nstages_override : ggml_cuda_fattn_mma_get_nstages(DKQ, DV, ncols1, ncols2, use_sparse); - constexpr bool swz = ggml_cuda_fattn_mma_get_swizzled(DKQ, DV, ncols1, ncols2); - constexpr int stride_tile_K = ggml_cuda_fattn_mma_get_stride_tile(nbatch_K2, swz); -@@ -1195,7 +1217,8 @@ template struct mma_tile_sizes { + constexpr bool swz = ggml_cuda_fattn_mma_get_swizzled(DKQ, DV, ncols); + constexpr int stride_tile_K = swz ? nbatch_K2 : nbatch_K2 + 4; +@@ -1186,7 +1208,8 @@ template struct mma_tile_sizes { }; #endif // defined(TURING_MMA_AVAILABLE) @@ -101,7 +130,7 @@ index 449a77c5..308218f1 100644 static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( const float2 * const __restrict__ Q_f2, const half2 * const __restrict__ K_h2, -@@ -1241,7 +1264,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( +@@ -1232,7 +1255,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( constexpr int nbatch_V2 = ggml_cuda_fattn_mma_get_nbatch_V2 (DKQ, DV, ncols); constexpr int nbatch_combine = ggml_cuda_fattn_mma_get_nbatch_combine(DKQ, DV, ncols); constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols); @@ -112,7 +141,7 @@ index 449a77c5..308218f1 100644 if (cols_per_warp > ncols) { NO_DEVICE_CODE; -@@ -1340,6 +1365,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( +@@ -1331,6 +1356,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( int kb0 = kb0_start; @@ -122,7 +151,7 @@ index 449a77c5..308218f1 100644 // Preload mask and K data for first iteration when using cp_async with multiple stages: if constexpr (nstages > 1) { static_assert(!use_sparse, "sparse gather not implemented for multi-stage loading"); -@@ -1347,16 +1375,54 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( +@@ -1338,16 +1366,54 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( constexpr bool use_cp_async = true; constexpr bool oob_check = false; constexpr int k_VKQ_sup = nbatch_fa; @@ -134,10 +163,10 @@ index 449a77c5..308218f1 100644 + flash_attn_ext_f16_load_mask + (mask_h, tile_mask, stride_mask, kb0*nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr); + } -+ flash_attn_ext_f16_load_tile ++ flash_attn_ext_f16_load_tile + (K_h2, tile_K, nbatch_K2, stride_K, kb0*nbatch_fa, k_VKQ_sup, nullptr); } -- flash_attn_ext_f16_load_tile +- flash_attn_ext_f16_load_tile - (K_h2, tile_K, nbatch_K2, stride_K, kb0*nbatch_fa, k_VKQ_sup, nullptr); } @@ -183,7 +212,7 @@ index 449a77c5..308218f1 100644 constexpr bool oob_check = true; for (; kb0 < kb0_stop-1; ++kb0) { constexpr bool last_iter = false; -@@ -1802,7 +1868,7 @@ static constexpr __host__ __device__ bool ggml_cuda_flash_attn_ext_mma_f16_may_u +@@ -1793,7 +1859,7 @@ static constexpr __host__ __device__ bool ggml_cuda_flash_attn_ext_mma_f16_may_u (DKQ == 256 && DV == 256 && ncols1 == 8 && ncols2 == 8); } @@ -192,7 +221,7 @@ index 449a77c5..308218f1 100644 __launch_bounds__(ggml_cuda_fattn_mma_get_nthreads(DKQ, DV, ncols1*ncols2), ggml_cuda_fattn_mma_get_occupancy(DKQ, DV, ncols1*ncols2)) static __global__ void flash_attn_ext_f16( const char * Q_ptr, -@@ -1945,12 +2011,12 @@ static __global__ void flash_attn_ext_f16( +@@ -1936,12 +2002,12 @@ static __global__ void flash_attn_ext_f16( constexpr bool is_fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer. if (kb0_start == 0) { constexpr bool needs_fixup = false; // CUDA block is working on an entire tile. @@ -207,7 +236,7 @@ index 449a77c5..308218f1 100644 (Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop); } -@@ -1994,7 +2060,7 @@ static __global__ void flash_attn_ext_f16( +@@ -1985,7 +2051,7 @@ static __global__ void flash_attn_ext_f16( constexpr bool is_fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks. constexpr bool needs_fixup = false; @@ -216,7 +245,7 @@ index 449a77c5..308218f1 100644 (Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop); #else -@@ -2013,6 +2079,47 @@ static __global__ void flash_attn_ext_f16( +@@ -2004,6 +2070,47 @@ static __global__ void flash_attn_ext_f16( bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(const int cc, const ggml_tensor * dst, const int ncols1, const int ncols2); @@ -227,18 +256,18 @@ index 449a77c5..308218f1 100644 + return true; +} + -+// Q columns per CUDA block of the long-sequence variant for long Q: 64 (the stock tile) or 128 (8 warps sharing each K/V tile). -+// GGML_CUDA_FA_LONGSEQ_NCOLS=64|128 overrides the per-architecture default. -+static int ggml_cuda_fattn_mma_longseq_ncols(const int cc) { ++// Q columns per CUDA block of the long-sequence variant for long Q: 64 (the stock tile, default) or 128 (8 warps sharing ++// each K/V tile, half the K/V traffic per FLOP). 128 is opt-in via GGML_CUDA_FA_LONGSEQ_NCOLS=128: it was measured faster at ++// MiniMax-H3 DiT shapes on A100 (sm80, 75 -> 70 ms) and L4 (sm89, 460 -> 365 ms) but splits the work differently from the ++// stock kernel, so its results are not bit-identical. ++static int ggml_cuda_fattn_mma_longseq_ncols(const int cc, const int DKQ) { + static const int env = [] { + const char * e = getenv("GGML_CUDA_FA_LONGSEQ_NCOLS"); + return e != nullptr ? atoi(e) : -1; + }(); -+ if (env == 64 || env == 128) { -+ return env; -+ } + GGML_UNUSED(cc); -+ return 64; ++ GGML_UNUSED(DKQ); ++ return env == 128 ? 128 : 64; +} + +static bool ggml_cuda_fattn_mma_use_longseq(const int cc, const ggml_tensor * dst) { @@ -264,7 +293,7 @@ index 449a77c5..308218f1 100644 template void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * KQV = dst; -@@ -2021,13 +2128,22 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml +@@ -2012,13 +2119,22 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml constexpr int ncols = ncols1 * ncols2; @@ -288,7 +317,7 @@ index 449a77c5..308218f1 100644 const int cols_per_warp = std::min(ncols, get_cols_per_warp(cc)); const int warp_size_host = ggml_cuda_info().devices[ctx.device].warp_size; -@@ -2061,6 +2177,21 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml +@@ -2052,6 +2168,40 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml #endif // defined(GGML_USE_HIP) fattn_kernel_t fattn_kernel; bool use_sparse = false; @@ -302,15 +331,34 @@ index 449a77c5..308218f1 100644 + CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total)); + shared_memory_limit_raised[id] = true; + } -+ launch_fattn -+ (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, use_sparse, warp_size_host); ++ if (ncols1 <= 64) { ++ // Same grid as the stock kernel of this tile size: identical stream-k seams, so results are bit-identical. ++ const int nstages_stock = ggml_cuda_fattn_mma_get_nstages(DKQ, DV, ncols1, ncols2, cc); ++ const size_t nbytes_KV_stock = nstages_stock <= 1 ? nbytes_shared_KV_1stage : nbytes_shared_KV_2stage; ++ const size_t nbytes_total_stock = std::max(nbytes_shared_combine, Q_in_reg ? ++ std::max(nbytes_shared_Q, nbytes_KV_stock + nbytes_shared_mask) : ++ nbytes_shared_Q + nbytes_KV_stock + nbytes_shared_mask); ++ fattn_kernel_t stock_kernel = flash_attn_ext_f16; ++ static bool stock_limit_raised[GGML_CUDA_MAX_DEVICES] = {false}; ++ if (!stock_limit_raised[id]) { ++ CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast(stock_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_total_stock)); ++ stock_limit_raised[id] = true; ++ } ++ launch_fattn ++ (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, use_sparse, warp_size_host, ++ nstages_stock == 2, stock_kernel, nbytes_total_stock); ++ } else { ++ launch_fattn ++ (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, use_sparse, warp_size_host, ++ nstages == 2); ++ } + return; + } + } if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; #if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) -@@ -2157,6 +2288,12 @@ DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 64) +@@ -2145,6 +2295,12 @@ DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256, 64) @@ -324,7 +372,7 @@ index 449a77c5..308218f1 100644 extern DECL_FATTN_MMA_F16_CASE(512, 512, 8, 2); extern DECL_FATTN_MMA_F16_CASE(512, 512, 16, 2); diff --git a/src/ggml-cuda/fattn.cu b/src/ggml-cuda/fattn.cu -index d1fcf58c..e3e34605 100644 +index 5fa3e530..9fb380e8 100644 --- a/src/ggml-cuda/fattn.cu +++ b/src/ggml-cuda/fattn.cu @@ -187,6 +187,15 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_con @@ -333,7 +381,7 @@ index d1fcf58c..e3e34605 100644 +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + if constexpr (ncols2 == 1 && DKQ == DV && (DKQ == 64 || DKQ == 128)) { -+ if (Q->ne[1] >= 1024 && ggml_cuda_fattn_mma_longseq_ncols(cc) == 128 && ggml_cuda_fattn_mma_use_longseq(cc, dst)) { ++ if (Q->ne[1] >= 1024 && ggml_cuda_fattn_mma_longseq_ncols(cc, DKQ) == 128 && ggml_cuda_fattn_mma_use_longseq(cc, dst)) { + ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst); + return; + } @@ -358,12 +406,12 @@ index 00000000..b9bf54a1 +DECL_FATTN_MMA_F16_CASE(128, 128, 128, 1); +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp -index e198d092..6a6954fe 100644 +index 5e2db805..83136355 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp -@@ -10840,6 +10840,21 @@ static std::vector> make_test_cases_eval() { - test_cases.emplace_back(new test_flash_attn_ext(72, 64, 4, {1, 1}, 256, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); - test_cases.emplace_back(new test_flash_attn_ext(64, 72, 4, {1, 1}, 256, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); +@@ -11710,6 +11710,21 @@ static std::vector> make_test_cases_eval() { + test_cases.emplace_back(new test_flash_attn_ext(65, 67, 4, {1, 1}, 17, 75, false, false, 0, 1.0f, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(65, 67, 4, {1, 1}, 113, 75, false, false, 0, 1.0f, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + // diffusion-shaped cases without a mask (CUDA long-sequence FA variant): KV lengths that are not a multiple + // of the KV tile (partial last tile), several Q tiles per head, and with/without GQA broadcast. @@ -383,182 +431,3 @@ index e198d092..6a6954fe 100644 // mixed quant and Q1_0 test cases test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_F16)); --- -2.43.0 - - -From 277581a8840cfd5bbf2335f379ef9be1f70e4754 Mon Sep 17 00:00:00 2001 -From: Daniel Han -Date: Thu, 1 Oct 2026 06:27:44 -0700 -Subject: [PATCH 2/3] ggml-cuda: per-architecture tile width for the - long-sequence FlashAttention variant - -Use 128 Q columns per block (8 warps sharing each K/V tile) by default for head -size 128 on sm80 and sm89, where it was measured faster at MiniMax-H3 DiT shapes -(A100: 75 -> 70 ms, L4: 460 -> 365 ms per 19108-token, 56-head call); keep 64 -elsewhere (neutral on sm100/sm120, slower for head size 64 on A100). -GGML_CUDA_FA_LONGSEQ_NCOLS=64|128 overrides. ---- - src/ggml-cuda/fattn-mma-f16.cuh | 13 ++++++++----- - src/ggml-cuda/fattn.cu | 2 +- - 2 files changed, 9 insertions(+), 6 deletions(-) - -diff --git a/src/ggml-cuda/fattn-mma-f16.cuh b/src/ggml-cuda/fattn-mma-f16.cuh -index 308218f1..c2d3196d 100644 ---- a/src/ggml-cuda/fattn-mma-f16.cuh -+++ b/src/ggml-cuda/fattn-mma-f16.cuh -@@ -2086,9 +2086,11 @@ static bool ggml_cuda_fattn_mma_longseq_arch_default(const int cc) { - return true; - } - --// Q columns per CUDA block of the long-sequence variant for long Q: 64 (the stock tile) or 128 (8 warps sharing each K/V tile). --// GGML_CUDA_FA_LONGSEQ_NCOLS=64|128 overrides the per-architecture default. --static int ggml_cuda_fattn_mma_longseq_ncols(const int cc) { -+// Q columns per CUDA block of the long-sequence variant for long Q: 64 (the stock tile) or 128 (8 warps sharing each K/V tile, -+// half the K/V traffic per FLOP). GGML_CUDA_FA_LONGSEQ_NCOLS=64|128 overrides the per-architecture default. -+// Measured at MiniMax-H3 DiT shapes (19108 tokens, 56 heads x 128): 128 is faster on A100 (sm80) and L4 (sm89), neutral on -+// RTX PRO 6000 Blackwell (sm120) and B200 (sm100), and slower for head size 64 on A100, so it is only the default there. -+static int ggml_cuda_fattn_mma_longseq_ncols(const int cc, const int DKQ) { - static const int env = [] { - const char * e = getenv("GGML_CUDA_FA_LONGSEQ_NCOLS"); - return e != nullptr ? atoi(e) : -1; -@@ -2096,8 +2098,9 @@ static int ggml_cuda_fattn_mma_longseq_ncols(const int cc) { - if (env == 64 || env == 128) { - return env; - } -- GGML_UNUSED(cc); -- return 64; -+ const int arch = ggml_cuda_highest_compiled_arch(cc); -+ return DKQ == 128 && (cc == GGML_CUDA_CC_AMPERE || cc == GGML_CUDA_CC_ADA_LOVELACE) && -+ (arch == GGML_CUDA_CC_AMPERE || arch == GGML_CUDA_CC_ADA_LOVELACE) ? 128 : 64; - } - - static bool ggml_cuda_fattn_mma_use_longseq(const int cc, const ggml_tensor * dst) { -diff --git a/src/ggml-cuda/fattn.cu b/src/ggml-cuda/fattn.cu -index e3e34605..f0fe5577 100644 ---- a/src/ggml-cuda/fattn.cu -+++ b/src/ggml-cuda/fattn.cu -@@ -189,7 +189,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_con - - #if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - if constexpr (ncols2 == 1 && DKQ == DV && (DKQ == 64 || DKQ == 128)) { -- if (Q->ne[1] >= 1024 && ggml_cuda_fattn_mma_longseq_ncols(cc) == 128 && ggml_cuda_fattn_mma_use_longseq(cc, dst)) { -+ if (Q->ne[1] >= 1024 && ggml_cuda_fattn_mma_longseq_ncols(cc, DKQ) == 128 && ggml_cuda_fattn_mma_use_longseq(cc, dst)) { - ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst); - return; - } --- -2.43.0 - - -From ad669abc9a620559691dab67db22a2ec41518127 Mon Sep 17 00:00:00 2001 -From: Daniel Han -Date: Thu, 1 Oct 2026 07:36:53 -0700 -Subject: [PATCH 3/3] ggml-cuda: long-sequence FlashAttention takes the stock - grid; 128-column tile opt-in only - -The stream-k grid is sized from occupancy, and the pipelined variant can have a -different occupancy than the stock kernel (two-stage shared memory), which -moves the stream-k seams and with them the order in which partial results are -combined (measured on L4: relative L2 6e-4 at the op, 36 dB PSNR end to end -over 4 diffusion steps). Size the grid from the stock kernel of the same tile -instead, so the pipelined variant is a bit-identical drop-in on every GPU. - -The 128-column tile cannot match the stock seams, so it is no longer a default -on any architecture; GGML_CUDA_FA_LONGSEQ_NCOLS=128 opts in. ---- - src/ggml-cuda/fattn-common.cuh | 8 +++++-- - src/ggml-cuda/fattn-mma-f16.cuh | 39 +++++++++++++++++++++++---------- - 2 files changed, 33 insertions(+), 14 deletions(-) - -diff --git a/src/ggml-cuda/fattn-common.cuh b/src/ggml-cuda/fattn-common.cuh -index 6d1ce52d..bc50e31f 100644 ---- a/src/ggml-cuda/fattn-common.cuh -+++ b/src/ggml-cuda/fattn-common.cuh -@@ -976,7 +976,10 @@ template - void launch_fattn( - ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, const int nwarps, const size_t nbytes_shared, - const int nbatch_fa, const bool need_f16_K, const bool need_f16_V, const bool stream_k, const bool use_sparse, -- const int warp_size = WARP_SIZE -+ const int warp_size = WARP_SIZE, -+ // Optional: size the grid from the occupancy of this kernel instead of fattn_kernel. A drop-in kernel variant launched -+ // with the grid of the kernel it replaces splits the work (stream-k seams) identically and so gives identical results. -+ fattn_kernel_t occupancy_kernel = nullptr, const size_t occupancy_nbytes_shared = 0 - ) { - constexpr int ncols = ncols1 * ncols2; - -@@ -1128,7 +1131,8 @@ void launch_fattn( - - const dim3 block_dim(warp_size, nwarps, 1); - int max_blocks_per_sm = 1; // Max. number of active blocks limited by occupancy. -- CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks_per_sm, fattn_kernel, block_dim.x * block_dim.y * block_dim.z, nbytes_shared)); -+ CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks_per_sm, occupancy_kernel ? occupancy_kernel : fattn_kernel, -+ block_dim.x * block_dim.y * block_dim.z, occupancy_kernel ? occupancy_nbytes_shared : nbytes_shared)); - GGML_ASSERT(max_blocks_per_sm > 0); - int parallel_blocks = max_blocks_per_sm; - -diff --git a/src/ggml-cuda/fattn-mma-f16.cuh b/src/ggml-cuda/fattn-mma-f16.cuh -index c2d3196d..dfc3182d 100644 ---- a/src/ggml-cuda/fattn-mma-f16.cuh -+++ b/src/ggml-cuda/fattn-mma-f16.cuh -@@ -2086,21 +2086,18 @@ static bool ggml_cuda_fattn_mma_longseq_arch_default(const int cc) { - return true; - } - --// Q columns per CUDA block of the long-sequence variant for long Q: 64 (the stock tile) or 128 (8 warps sharing each K/V tile, --// half the K/V traffic per FLOP). GGML_CUDA_FA_LONGSEQ_NCOLS=64|128 overrides the per-architecture default. --// Measured at MiniMax-H3 DiT shapes (19108 tokens, 56 heads x 128): 128 is faster on A100 (sm80) and L4 (sm89), neutral on --// RTX PRO 6000 Blackwell (sm120) and B200 (sm100), and slower for head size 64 on A100, so it is only the default there. -+// Q columns per CUDA block of the long-sequence variant for long Q: 64 (the stock tile, default) or 128 (8 warps sharing -+// each K/V tile, half the K/V traffic per FLOP). 128 is opt-in via GGML_CUDA_FA_LONGSEQ_NCOLS=128: it was measured faster at -+// MiniMax-H3 DiT shapes on A100 (sm80, 75 -> 70 ms) and L4 (sm89, 460 -> 365 ms) but splits the work differently from the -+// stock kernel, so its results are not bit-identical. - static int ggml_cuda_fattn_mma_longseq_ncols(const int cc, const int DKQ) { - static const int env = [] { - const char * e = getenv("GGML_CUDA_FA_LONGSEQ_NCOLS"); - return e != nullptr ? atoi(e) : -1; - }(); -- if (env == 64 || env == 128) { -- return env; -- } -- const int arch = ggml_cuda_highest_compiled_arch(cc); -- return DKQ == 128 && (cc == GGML_CUDA_CC_AMPERE || cc == GGML_CUDA_CC_ADA_LOVELACE) && -- (arch == GGML_CUDA_CC_AMPERE || arch == GGML_CUDA_CC_ADA_LOVELACE) ? 128 : 64; -+ GGML_UNUSED(cc); -+ GGML_UNUSED(DKQ); -+ return env == 128 ? 128 : 64; - } - - static bool ggml_cuda_fattn_mma_use_longseq(const int cc, const ggml_tensor * dst) { -@@ -2190,8 +2187,26 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml - CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total)); - shared_memory_limit_raised[id] = true; - } -- launch_fattn -- (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, use_sparse, warp_size_host); -+ if (ncols1 <= 64) { -+ // Same grid as the stock kernel of this tile size: identical stream-k seams, so results are bit-identical. -+ const int nstages_stock = ggml_cuda_fattn_mma_get_nstages(DKQ, DV, ncols1, ncols2, cc); -+ const size_t nbytes_KV_stock = nstages_stock <= 1 ? nbytes_shared_KV_1stage : nbytes_shared_KV_2stage; -+ const size_t nbytes_total_stock = std::max(nbytes_shared_combine, Q_in_reg ? -+ std::max(nbytes_shared_Q, nbytes_KV_stock + nbytes_shared_mask) : -+ nbytes_shared_Q + nbytes_KV_stock + nbytes_shared_mask); -+ fattn_kernel_t stock_kernel = flash_attn_ext_f16; -+ static bool stock_limit_raised[GGML_CUDA_MAX_DEVICES] = {false}; -+ if (!stock_limit_raised[id]) { -+ CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast(stock_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_total_stock)); -+ stock_limit_raised[id] = true; -+ } -+ launch_fattn -+ (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, use_sparse, warp_size_host, -+ stock_kernel, nbytes_total_stock); -+ } else { -+ launch_fattn -+ (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, use_sparse, warp_size_host); -+ } - return; - } - } --- -2.43.0 - diff --git a/scripts/unsloth/ggml-patches/0003-ggml-cuda-h3-vae-epilogue-fusion.patch b/scripts/unsloth/ggml-patches/0003-ggml-cuda-h3-vae-epilogue-fusion.patch index e227077e2..0d086f621 100644 --- a/scripts/unsloth/ggml-patches/0003-ggml-cuda-h3-vae-epilogue-fusion.patch +++ b/scripts/unsloth/ggml-patches/0003-ggml-cuda-h3-vae-epilogue-fusion.patch @@ -23,11 +23,14 @@ the 256-thread launch. src/ggml-cuda/norm.cu | 63 ++++++++++++ 2 files changed, 262 insertions(+), 1 deletion(-) +[Ported onto leejet/ggml dc998771 (stable-diffusion.cpp master-982): the diff below is + the whole patch as one diff, regenerated from the port.] +--- diff --git a/src/ggml-cuda/ggml-cuda.cu b/src/ggml-cuda/ggml-cuda.cu -index a153de57..72d090e3 100644 +index 8a295c19..74f3373a 100644 --- a/src/ggml-cuda/ggml-cuda.cu +++ b/src/ggml-cuda/ggml-cuda.cu -@@ -2014,8 +2014,73 @@ struct batched_mul_mat_traits { +@@ -2061,8 +2061,73 @@ struct batched_mul_mat_traits { static inline auto convert_nc(ggml_type src_type) { return ggml_get_to_fp16_nc_cuda(src_type); } }; @@ -102,7 +105,7 @@ index a153de57..72d090e3 100644 using traits = batched_mul_mat_traits; using cuda_t = typename traits::cuda_type; -@@ -2223,8 +2288,17 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const +@@ -2270,8 +2335,17 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const // Convert output back to F32 if needed if (cu_data_type != CUDA_R_32F) { @@ -120,7 +123,7 @@ index a153de57..72d090e3 100644 } } -@@ -4111,6 +4185,123 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, +@@ -4336,6 +4410,123 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, } // try and fuse nodes and return the number of nodes to skip @@ -153,7 +156,7 @@ index a153de57..72d090e3 100644 + return true; + } + const int64_t ne11 = src1->ne[1]; -+ return !ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, ne11) && ++ return !ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11) && + !ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11, /*mul_mat_id =*/ false); +} + @@ -243,8 +246,8 @@ index a153de57..72d090e3 100644 + static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { - static bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION")); -@@ -4825,6 +5016,13 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph + if (ggml_cuda_fusion_disabled()) { +@@ -5083,6 +5274,13 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph return 2; } @@ -259,10 +262,10 @@ index a153de57..72d090e3 100644 ggml_cuda_op_rms_norm_fused_add(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]); return 2; diff --git a/src/ggml-cuda/norm.cu b/src/ggml-cuda/norm.cu -index c3758cd5..9c9385f8 100644 +index d804b933..854d47b7 100644 --- a/src/ggml-cuda/norm.cu +++ b/src/ggml-cuda/norm.cu -@@ -301,10 +301,68 @@ static void group_norm_f32_cuda( +@@ -332,12 +332,70 @@ static void group_norm_f32_cuda( } } @@ -282,60 +285,62 @@ index c3758cd5..9c9385f8 100644 +#endif +} + -+template ++template +static void rms_norm_f32_launch_small( -+ const float * x, float * dst, const int ncols, const dim3 blocks_num, ++ const float * x, float * dst, const int ncols, const int nchannels, const int nsamples, const dim3 blocks_num, + const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, + const float * mul, const int64_t mul_stride_row, const int64_t mul_stride_channel, const int64_t mul_stride_sample, + const uint3 mul_ncols_packed, const uint3 mul_nrows_packed, const uint3 mul_nchannels_packed, const uint3 mul_nsamples_packed, -+ cudaStream_t stream) { ++ cudaStream_t stream, const float scale_out) { + const dim3 block_dims(block_size, 1, 1); + const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, block_size > WARP_SIZE ? 32 * sizeof(float) : 0, stream}; -+ ggml_cuda_kernel_launch(rms_norm_f32, launch_params, -+ x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel, ++ ggml_cuda_kernel_launch(rms_norm_f32, launch_params, ++ x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel, + mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, -+ nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0)); ++ nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), scale_out); +} + -+template ++template +static bool rms_norm_f32_try_small( -+ const float * x, float * dst, const int ncols, const dim3 blocks_num, ++ const float * x, float * dst, const int ncols, const int nchannels, const int nsamples, const dim3 blocks_num, + const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, + const float * mul, const int64_t mul_stride_row, const int64_t mul_stride_channel, const int64_t mul_stride_sample, + const uint3 mul_ncols_packed, const uint3 mul_nrows_packed, const uint3 mul_nchannels_packed, const uint3 mul_nsamples_packed, -+ cudaStream_t stream) { ++ cudaStream_t stream, const float scale_out = 1.0f) { + if (ncols > 128 || !rms_norm_small_rows_enabled()) { + return false; + } + if (ncols <= 32) { -+ rms_norm_f32_launch_small<32, do_multiply>(x, dst, ncols, blocks_num, stride_row, stride_channel, stride_sample, eps, mul, -+ mul_stride_row, mul_stride_channel, mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, stream); ++ rms_norm_f32_launch_small<32, do_multiply, do_scale>(x, dst, ncols, nchannels, nsamples, blocks_num, stride_row, stride_channel, stride_sample, eps, mul, ++ mul_stride_row, mul_stride_channel, mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, stream, scale_out); + } else if (ncols <= 64) { -+ rms_norm_f32_launch_small<64, do_multiply>(x, dst, ncols, blocks_num, stride_row, stride_channel, stride_sample, eps, mul, -+ mul_stride_row, mul_stride_channel, mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, stream); ++ rms_norm_f32_launch_small<64, do_multiply, do_scale>(x, dst, ncols, nchannels, nsamples, blocks_num, stride_row, stride_channel, stride_sample, eps, mul, ++ mul_stride_row, mul_stride_channel, mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, stream, scale_out); + } else { -+ rms_norm_f32_launch_small<128, do_multiply>(x, dst, ncols, blocks_num, stride_row, stride_channel, stride_sample, eps, mul, -+ mul_stride_row, mul_stride_channel, mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, stream); ++ rms_norm_f32_launch_small<128, do_multiply, do_scale>(x, dst, ncols, nchannels, nsamples, blocks_num, stride_row, stride_channel, stride_sample, eps, mul, ++ mul_stride_row, mul_stride_channel, mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, stream, scale_out); + } + return true; +} + + template static void rms_norm_f32_cuda( const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, - const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) { - const dim3 blocks_num(nrows, nchannels, nsamples); -+ if (rms_norm_f32_try_small(x, dst, ncols, blocks_num, stride_row, stride_channel, stride_sample, eps, -+ nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), stream)) { + const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream, + const float scale_out = 1.0f) { + const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX)); ++ if (rms_norm_f32_try_small(x, dst, ncols, nchannels, nsamples, blocks_num, stride_row, stride_channel, stride_sample, eps, ++ nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), stream, scale_out)) { + return; + } if (ncols < 1024) { const dim3 block_dims(256, 1, 1); const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream}; -@@ -360,6 +418,11 @@ static void rms_norm_mul_f32_cuda(const float * x, +@@ -393,6 +451,11 @@ static void rms_norm_mul_f32_cuda(const float * x, const uint3 mul_nrows_packed = init_fastdiv_values(mul_nrows); const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels); const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples); -+ if (rms_norm_f32_try_small(x, dst, ncols, blocks_num, stride_row, stride_channel, stride_sample, eps, ++ if (rms_norm_f32_try_small(x, dst, ncols, nchannels, nsamples, blocks_num, stride_row, stride_channel, stride_sample, eps, + mul, mul_stride_row, mul_stride_channel, mul_stride_sample, mul_ncols_packed, mul_nrows_packed, + mul_nchannels_packed, mul_nsamples_packed, stream)) { + return; @@ -343,6 +348,3 @@ index c3758cd5..9c9385f8 100644 if (ncols < 1024) { const dim3 block_dims(256, 1, 1); const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream}; --- -2.43.0 - diff --git a/scripts/unsloth/ggml-patches/0004-ggml-h3-dit-fused-ops.patch b/scripts/unsloth/ggml-patches/0004-ggml-h3-dit-fused-ops.patch index 29592cf66..386d09a04 100644 --- a/scripts/unsloth/ggml-patches/0004-ggml-h3-dit-fused-ops.patch +++ b/scripts/unsloth/ggml-patches/0004-ggml-h3-dit-fused-ops.patch @@ -42,8 +42,11 @@ partial and full rotation, F32/F16 outputs, chained segments and the scales. create mode 100644 src/ggml-cuda/rope-pe.cu create mode 100644 src/ggml-cuda/rope-pe.cuh +[Ported onto leejet/ggml dc998771 (stable-diffusion.cpp master-982): the diff below is + the whole patch as one diff, regenerated from the port.] +--- diff --git a/include/ggml.h b/include/ggml.h -index 761d3f28..8c92967a 100644 +index 49a6751d..bbafe175 100644 --- a/include/ggml.h +++ b/include/ggml.h @@ -607,6 +607,9 @@ extern "C" { @@ -56,7 +59,7 @@ index 761d3f28..8c92967a 100644 GGML_OP_COUNT, }; -@@ -2551,6 +2554,65 @@ extern "C" { +@@ -2556,6 +2559,65 @@ extern "C" { float scale, float tau); @@ -123,7 +126,7 @@ index 761d3f28..8c92967a 100644 struct ggml_tensor * a, enum ggml_prec prec), diff --git a/src/ggml-backend-meta.cpp b/src/ggml-backend-meta.cpp -index 98ba5020..426eaa86 100644 +index fb447e97..899c8ad4 100644 --- a/src/ggml-backend-meta.cpp +++ b/src/ggml-backend-meta.cpp @@ -153,7 +153,8 @@ static ggml_backend_buffer_type_t ggml_backend_meta_device_get_host_buffer_type( @@ -137,10 +140,10 @@ index 98ba5020..426eaa86 100644 } const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context; diff --git a/src/ggml-cpu/ggml-cpu.c b/src/ggml-cpu/ggml-cpu.c -index 92de0631..d4dcefdf 100644 +index fd55589f..30b8cbb5 100644 --- a/src/ggml-cpu/ggml-cpu.c +++ b/src/ggml-cpu/ggml-cpu.c -@@ -2224,6 +2224,18 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm +@@ -2220,6 +2220,18 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_glu(params, tensor); } break; @@ -159,7 +162,7 @@ index 92de0631..d4dcefdf 100644 case GGML_OP_GET_REL_POS: { ggml_compute_forward_get_rel_pos(params, tensor); -@@ -2558,6 +2570,9 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { +@@ -2554,6 +2566,9 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_ROPE_BACK: case GGML_OP_ROPE_APPLY: case GGML_OP_ADD_REL_POS: @@ -170,7 +173,7 @@ index 92de0631..d4dcefdf 100644 n_tasks = n_threads; } break; diff --git a/src/ggml-cpu/ops.cpp b/src/ggml-cpu/ops.cpp -index b68afce1..f4ca82e7 100644 +index e8ef14f2..94333d30 100644 --- a/src/ggml-cpu/ops.cpp +++ b/src/ggml-cpu/ops.cpp @@ -12,6 +12,7 @@ @@ -181,7 +184,7 @@ index b68afce1..f4ca82e7 100644 struct ggml_fp8_e4m3_cpu { uint8_t value; -@@ -12380,3 +12381,156 @@ void ggml_compute_forward_quantize_i8_convrot(const ggml_compute_params * params +@@ -12686,3 +12687,156 @@ void ggml_compute_forward_quantize_i8_convrot(const ggml_compute_params * params memset(dst_data + rows * dst->nb[1], 0, (size_t)(rows_padded - rows) * dst->nb[1]); } } @@ -353,7 +356,7 @@ index 19d7075e..d731a958 100644 void ggml_compute_forward_add_rel_pos(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_rwkv_wkv6(const struct ggml_compute_params * params, struct ggml_tensor * dst); diff --git a/src/ggml-cuda/ggml-cuda.cu b/src/ggml-cuda/ggml-cuda.cu -index 72d090e3..7e98d729 100644 +index cd0cee80..5c799ccd 100644 --- a/src/ggml-cuda/ggml-cuda.cu +++ b/src/ggml-cuda/ggml-cuda.cu @@ -47,6 +47,8 @@ @@ -365,7 +368,7 @@ index 72d090e3..7e98d729 100644 #include "ggml-cuda/snake.cuh" #include "ggml-cuda/softcap.cuh" #include "ggml-cuda/softmax.cuh" -@@ -3123,6 +3125,15 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg +@@ -3253,6 +3255,15 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_SOL_ATTN: ggml_cuda_sol_attn(ctx, dst); break; @@ -381,7 +384,7 @@ index 72d090e3..7e98d729 100644 case GGML_OP_CROSS_ENTROPY_LOSS: ggml_cuda_cross_entropy_loss(ctx, dst); break; -@@ -6492,6 +6503,12 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g +@@ -6784,6 +6795,12 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return ggml_cuda_sage_attn_supported(dev_ctx->device, op); case GGML_OP_SOL_ATTN: return ggml_cuda_sol_attn_supported(dev_ctx->device, op); @@ -701,10 +704,10 @@ index 00000000..b68b6af4 + +bool ggml_cuda_rope_pe_permute_supported(const ggml_tensor * op); diff --git a/src/ggml.c b/src/ggml.c -index 562b6eb9..165514d9 100644 +index da9ccada..a9f0b47d 100644 --- a/src/ggml.c +++ b/src/ggml.c -@@ -1118,9 +1118,12 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { +@@ -1129,9 +1129,12 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "SAGE_ATTN", "SOL_ATTN", "ROPE_APPLY", @@ -718,7 +721,7 @@ index 562b6eb9..165514d9 100644 static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "none", -@@ -1238,9 +1241,12 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { +@@ -1249,9 +1252,12 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "sage_attn(x,y,z)", "sol_attn(x,y,z)", "rope_apply(x,pe)", @@ -732,7 +735,7 @@ index 562b6eb9..165514d9 100644 static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); -@@ -5699,6 +5705,125 @@ struct ggml_tensor* ggml_sol_attn( +@@ -5721,6 +5727,125 @@ struct ggml_tensor* ggml_sol_attn( return result; } @@ -859,10 +862,10 @@ index 562b6eb9..165514d9 100644 struct ggml_tensor * a, enum ggml_prec prec) { diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp -index 6a6954fe..6b49765d 100644 +index 83136355..fdda5dc7 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp -@@ -2188,6 +2188,118 @@ struct test_glu : public test_case { +@@ -2192,6 +2192,118 @@ struct test_glu : public test_case { } }; @@ -981,7 +984,7 @@ index 6a6954fe..6b49765d 100644 struct test_glu_split : public test_case { const ggml_glu_op op; const ggml_type type; -@@ -9106,6 +9218,33 @@ static std::vector> make_test_cases_eval() { +@@ -9759,6 +9871,33 @@ static std::vector> make_test_cases_eval() { } } @@ -1012,9 +1015,6 @@ index 6a6954fe..6b49765d 100644 + } + } + - for (int v : {0, 1}) { - for (float alpha : {.5f, 1.702f}) { - for (float limit : {2.0f, 7.0f}) { --- -2.43.0 - + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (int v : {0, 1}) { + for (float alpha : {.5f, 1.702f}) { diff --git a/scripts/unsloth/ggml-patches/0005-ggml-rope-pe-permute-rms.patch b/scripts/unsloth/ggml-patches/0005-ggml-rope-pe-permute-rms.patch index ce6492e9a..e5f2231ea 100644 --- a/scripts/unsloth/ggml-patches/0005-ggml-rope-pe-permute-rms.patch +++ b/scripts/unsloth/ggml-patches/0005-ggml-rope-pe-permute-rms.patch @@ -18,11 +18,14 @@ reported unsupported and callers keep the separate norm. test-backend-ops cases tests/test-backend-ops.cpp | 15 +++-- 5 files changed, 174 insertions(+), 5 deletions(-) +[Ported onto leejet/ggml dc998771 (stable-diffusion.cpp master-982): the diff below is + the whole patch as one diff, regenerated from the port.] +--- diff --git a/include/ggml.h b/include/ggml.h -index 8c92967a..41ac28bb 100644 +index bbafe175..f7ef1573 100644 --- a/include/ggml.h +++ b/include/ggml.h -@@ -2573,6 +2573,18 @@ extern "C" { +@@ -2578,6 +2578,18 @@ extern "C" { float scale, enum ggml_type type); @@ -42,10 +45,10 @@ index 8c92967a..41ac28bb 100644 // different modulation vectors. Writes rows [row0, row1) along dim 1 (all of dims 2 and 3) of the // result and leaves every other row as it is in `acc`: chain one call per segment, passing the diff --git a/src/ggml-cpu/ops.cpp b/src/ggml-cpu/ops.cpp -index f4ca82e7..bdd36aea 100644 +index 94333d30..ebe6e202 100644 --- a/src/ggml-cpu/ops.cpp +++ b/src/ggml-cpu/ops.cpp -@@ -12395,6 +12395,8 @@ void ggml_compute_forward_rope_pe_permute( +@@ -12701,6 +12701,8 @@ void ggml_compute_forward_rope_pe_permute( const int n_rot = ggml_get_op_params_i32(dst, 0); const float scale = ggml_get_op_params_f32(dst, 1); @@ -54,7 +57,7 @@ index f4ca82e7..bdd36aea 100644 const int half = n_rot / 2; const int64_t D = src0->ne[0]; -@@ -12409,7 +12411,7 @@ void ggml_compute_forward_rope_pe_permute( +@@ -12715,7 +12717,7 @@ void ggml_compute_forward_rope_pe_permute( // Same rounding points as the unfused graph: each product, the sum, the scale and the // conversion are separate passes, so no contraction into fused multiply-adds can happen. @@ -63,7 +66,7 @@ index f4ca82e7..bdd36aea 100644 for (int64_t ir = ir0; ir < ir1; ++ir) { const int64_t h = ir % H; -@@ -12418,6 +12420,19 @@ void ggml_compute_forward_rope_pe_permute( +@@ -12724,6 +12726,19 @@ void ggml_compute_forward_rope_pe_permute( const float * x = (const float *) ((const char *) src0->data + h * src0->nb[1] + l * src0->nb[2] + n * src0->nb[3]); @@ -228,10 +231,10 @@ index 3afe4423..b6dd7a69 100644 return pe == nullptr || (pe->type == GGML_TYPE_F32 && ggml_is_contiguous(pe)); } diff --git a/src/ggml.c b/src/ggml.c -index 165514d9..1fe59a0e 100644 +index a9f0b47d..1dcc8a0f 100644 --- a/src/ggml.c +++ b/src/ggml.c -@@ -5736,6 +5736,21 @@ struct ggml_tensor * ggml_rope_pe_permute( +@@ -5758,6 +5758,21 @@ struct ggml_tensor * ggml_rope_pe_permute( return result; } @@ -254,10 +257,10 @@ index 165514d9..1fe59a0e 100644 struct ggml_context * ctx, struct ggml_tensor * acc, diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp -index 6b49765d..af772ae9 100644 +index fdda5dc7..108da4a2 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp -@@ -2195,15 +2195,16 @@ struct test_rope_pe_permute : public test_case { +@@ -2199,15 +2199,16 @@ struct test_rope_pe_permute : public test_case { const int n_rot; const float scale; const bool view; // a is a strided view (one third of a fused qkv projection) @@ -277,7 +280,7 @@ index 6b49765d..af772ae9 100644 ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a; -@@ -2222,7 +2223,8 @@ struct test_rope_pe_permute : public test_case { +@@ -2226,7 +2227,8 @@ struct test_rope_pe_permute : public test_case { pe = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 2, 2, n_rot / 2, ne_a[2]); ggml_set_name(pe, "pe"); } @@ -287,7 +290,7 @@ index 6b49765d..af772ae9 100644 ggml_set_name(out, "out"); return out; } -@@ -9226,6 +9228,11 @@ static std::vector> make_test_cases_eval() { +@@ -9879,6 +9881,11 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_rope_pe_permute(type, {128, 4, 9, 1}, 0, scale, view)); test_cases.emplace_back(new test_rope_pe_permute(type, {64, 3, 7, 2}, 32, scale, view)); test_cases.emplace_back(new test_rope_pe_permute(type, {200, 2, 3, 1}, 100, scale, view)); @@ -299,6 +302,3 @@ index 6b49765d..af772ae9 100644 } } } --- -2.43.0 - diff --git a/scripts/unsloth/ggml-patches/0006-ggml-cuda-cudnn-attention.patch b/scripts/unsloth/ggml-patches/0006-ggml-cuda-cudnn-attention.patch index 68aecc56f..73fb6cd4c 100644 --- a/scripts/unsloth/ggml-patches/0006-ggml-cuda-cudnn-attention.patch +++ b/scripts/unsloth/ggml-patches/0006-ggml-cuda-cudnn-attention.patch @@ -49,11 +49,14 @@ lengths, GQA with batch, F32 K/V, permuted Q) and unmasked perf cases. create mode 100644 src/ggml-cuda/cudnn-attn.cu create mode 100644 src/ggml-cuda/cudnn-attn.cuh +[Ported onto leejet/ggml dc998771 (stable-diffusion.cpp master-982): the diff below is + the whole patch as one diff, regenerated from the port.] +--- diff --git a/CMakeLists.txt b/CMakeLists.txt -index 752dabbb..91e94ba3 100644 +index 35252a83..da110ce1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -208,6 +208,9 @@ set (GGML_CUDA_FA_QUANTS "q4_0-q4_0;q8_0-q8_0;f16-f16;bf16-bf16" CACHE STRING +@@ -209,6 +209,9 @@ set (GGML_CUDA_FA_QUANTS "q4_0-q4_0;q8_0-q8_0;f16-f16;bf16-bf16" CACHE STRING "ggml: FlashAttention K-V type combinations to compile, \"all\" or a list such as \"q8_0-q8_0;q8_0-q4_0\"") option(GGML_CUDA_GRAPHS "ggml: use CUDA graphs (llama.cpp only)" ${GGML_CUDA_GRAPHS_DEFAULT}) option(GGML_CUDA_NCCL "ggml: use NVIDIA Collective Comm. Library" ON) @@ -64,10 +67,10 @@ index 752dabbb..91e94ba3 100644 "ggml: cuda link binary compression mode; requires cuda 12.8+") set_property(CACHE GGML_CUDA_COMPRESSION_MODE PROPERTY STRINGS "none;speed;balance;size") diff --git a/src/ggml-cuda/CMakeLists.txt b/src/ggml-cuda/CMakeLists.txt -index d272f6f7..ba2a2080 100644 +index 37aa6d94..7c2fb7dd 100644 --- a/src/ggml-cuda/CMakeLists.txt +++ b/src/ggml-cuda/CMakeLists.txt -@@ -115,11 +115,60 @@ if (CUDAToolkit_FOUND) +@@ -127,11 +127,60 @@ if (CUDAToolkit_FOUND) ggml_cuda_fattn_vec_instances(${CMAKE_CURRENT_SOURCE_DIR} SRCS) list(APPEND GGML_SOURCES_CUDA ${SRCS}) @@ -852,7 +855,7 @@ index 00000000..067bea63 +// so the caller runs its own kernel. +bool ggml_cuda_cudnn_attn(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/src/ggml-cuda/fattn.cu b/src/ggml-cuda/fattn.cu -index f0fe5577..2eda0ccd 100644 +index 9fb380e8..84c4e653 100644 --- a/src/ggml-cuda/fattn.cu +++ b/src/ggml-cuda/fattn.cu @@ -4,6 +4,7 @@ @@ -863,7 +866,7 @@ index f0fe5577..2eda0ccd 100644 #if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) // one list per group of ncols1 queries: a column is selected if any query of the group can see it -@@ -758,11 +759,15 @@ size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * d +@@ -760,11 +761,15 @@ size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * d const ggml_cuda_flash_attn_ext_f16_extra_data f16_extra = ggml_cuda_flash_attn_ext_get_f16_extra_data(dst, need_f16_K, need_f16_V); @@ -881,10 +884,10 @@ index f0fe5577..2eda0ccd 100644 case BEST_FATTN_KERNEL_NONE: GGML_ABORT("fatal error"); diff --git a/src/ggml-cuda/ggml-cuda.cu b/src/ggml-cuda/ggml-cuda.cu -index 7e98d729..831d0de8 100644 +index 5c799ccd..f19aba44 100644 --- a/src/ggml-cuda/ggml-cuda.cu +++ b/src/ggml-cuda/ggml-cuda.cu -@@ -922,7 +922,7 @@ static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_t +@@ -961,7 +961,7 @@ static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_t ? ggml_cuda_flash_attn_ext_get_alloc_size(buft_ctx->device, tensor) : ggml_nbytes(tensor); if (tensor->op == GGML_OP_SAGE_ATTN) { @@ -947,12 +950,12 @@ index 126c7a74..e2648857 100644 #endif diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp -index af772ae9..778bed20 100644 +index 108da4a2..62be5695 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp -@@ -11099,6 +11099,21 @@ static std::vector> make_test_cases_eval() { - test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {2, 1}, 1024, 32, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); - test_cases.emplace_back(new test_flash_attn_ext(512, 512, 4, {2, 1}, 1024, 4, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); +@@ -12008,6 +12008,21 @@ static std::vector> make_test_cases_eval() { + test_cases.emplace_back(new test_flash_attn_ext_large_logits(256, 256, 4, {4, 1}, 1024, 75, true, false, 0, 10.0f, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext_large_logits(256, 256, 4, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16)); + // unmasked F16 K/V with the MiniMax-H3 DiT head layout (42 heads x 128), the shape class the CUDA + // cuDNN attention path takes: odd and tile-aligned lengths, GQA with batch, F32 K/V, head size 64. @@ -972,8 +975,8 @@ index af772ae9..778bed20 100644 test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, { 10, 5, 4, 3})); test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, {30000, 1, 1, 1})); test_cases.emplace_back(new test_cross_entropy_loss_back(GGML_TYPE_F32, { 10, 5, 4, 3})); -@@ -11527,6 +11542,16 @@ static std::vector> make_test_cases_perf() { - test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 2048)); +@@ -12469,6 +12484,16 @@ static std::vector> make_test_cases_perf() { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, false, 0)); } + // unmasked diffusion attention: MiniMax-H3 DiT (19315 tokens x 56 heads at 960x544x124; 22440 x 42), @@ -989,6 +992,3 @@ index af772ae9..778bed20 100644 test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 4, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); --- -2.43.0 - diff --git a/scripts/unsloth/ggml-patches/0007-ggml-cuda-generic-dit-vae.patch b/scripts/unsloth/ggml-patches/0007-ggml-cuda-generic-dit-vae.patch index b7fd0dfcb..79a61e549 100644 --- a/scripts/unsloth/ggml-patches/0007-ggml-cuda-generic-dit-vae.patch +++ b/scripts/unsloth/ggml-patches/0007-ggml-cuda-generic-dit-vae.patch @@ -1,8 +1,8 @@ diff --git a/include/ggml.h b/include/ggml.h -index 41ac28bb..fa83c5e6 100644 +index f7ef1573..bec0b7b6 100644 --- a/include/ggml.h +++ b/include/ggml.h -@@ -2576,6 +2576,14 @@ extern "C" { +@@ -2581,6 +2581,14 @@ extern "C" { // ggml_rope_pe_permute of ggml_rms_norm(a, eps) (no weight): every head row of `a` is // RMS-normalised first, with the same arithmetic as GGML_OP_RMS_NORM on the same backend, so // the result matches the two-op chain bit for bit where the backend supports it. @@ -18,10 +18,10 @@ index 41ac28bb..fa83c5e6 100644 struct ggml_context * ctx, struct ggml_tensor * a, diff --git a/src/ggml-cpu/ops.cpp b/src/ggml-cpu/ops.cpp -index bdd36aea..2883f5df 100644 +index ebe6e202..f7d6b8f7 100644 --- a/src/ggml-cpu/ops.cpp +++ b/src/ggml-cpu/ops.cpp -@@ -12397,6 +12397,7 @@ void ggml_compute_forward_rope_pe_permute( +@@ -12703,6 +12703,7 @@ void ggml_compute_forward_rope_pe_permute( const float scale = ggml_get_op_params_f32(dst, 1); const bool norm = ggml_get_op_params_i32(dst, 2) != 0; const float eps = ggml_get_op_params_f32(dst, 3); @@ -29,7 +29,7 @@ index bdd36aea..2883f5df 100644 const int half = n_rot / 2; const int64_t D = src0->ne[0]; -@@ -12435,17 +12436,28 @@ void ggml_compute_forward_rope_pe_permute( +@@ -12741,17 +12742,28 @@ void ggml_compute_forward_rope_pe_permute( if (n_rot > 0) { const float * t = (const float *) pe->data + l * (int64_t) half * 4; @@ -69,7 +69,7 @@ index bdd36aea..2883f5df 100644 if (D > n_rot) { memcpy(y.data() + n_rot, x + n_rot, (D - n_rot) * sizeof(float)); diff --git a/src/ggml-cuda/ggml-cuda.cu b/src/ggml-cuda/ggml-cuda.cu -index 831d0de8..8f4e6f3d 100644 +index f19aba44..05ac1936 100644 --- a/src/ggml-cuda/ggml-cuda.cu +++ b/src/ggml-cuda/ggml-cuda.cu @@ -37,6 +37,9 @@ @@ -82,7 +82,7 @@ index 831d0de8..8f4e6f3d 100644 #include "ggml-cuda/opt-step-adamw.cuh" #include "ggml-cuda/opt-step-sgd.cuh" #include "ggml-cuda/out-prod.cuh" -@@ -4313,6 +4316,106 @@ static int ggml_cuda_try_fuse_cublas_epilogue(ggml_backend_cuda_context & ctx, g +@@ -4538,6 +4541,106 @@ static int ggml_cuda_try_fuse_cublas_epilogue(ggml_backend_cuda_context & ctx, g return consumed; } @@ -188,10 +188,10 @@ index 831d0de8..8f4e6f3d 100644 + static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { - static bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION")); -@@ -4322,6 +4425,20 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph - - ggml_tensor * node = cgraph->nodes[i]; + if (ggml_cuda_fusion_disabled()) { +@@ -4566,6 +4669,20 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph + } + } + if (node->op == GGML_OP_GROUP_NORM) { + const int consumed = ggml_cuda_try_fuse_group_norm(*cuda_ctx, cgraph, i); @@ -210,7 +210,7 @@ index 831d0de8..8f4e6f3d 100644 if (node->op == GGML_OP_MUL) { ggml_cuda_moe_weighted_reduction_match match; if (ggml_cuda_match_moe_weighted_reduction(cgraph, i, match)) { -@@ -4334,6 +4451,13 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph +@@ -4578,6 +4695,13 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph } } @@ -898,7 +898,7 @@ index 00000000..da57e590 +// GGML_CUDA_NORM_CHANNEL=0 turns it off. +int ggml_cuda_try_fuse_rms_norm_channel(ggml_backend_cuda_context & ctx, ggml_cgraph * cgraph, int i); diff --git a/src/ggml-cuda/norm.cu b/src/ggml-cuda/norm.cu -index 9c9385f8..ae2def32 100644 +index 854d47b7..7ed01b7b 100644 --- a/src/ggml-cuda/norm.cu +++ b/src/ggml-cuda/norm.cu @@ -1,4 +1,5 @@ @@ -907,7 +907,7 @@ index 9c9385f8..ae2def32 100644 #include template -@@ -534,6 +535,11 @@ void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) +@@ -567,6 +568,11 @@ void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) memcpy(&eps, dst->op_params + 1, sizeof(float)); GGML_ASSERT(eps >= 0.0f); @@ -1099,7 +1099,7 @@ index b6dd7a69..2fe9a599 100644 } diff --git a/src/ggml-cuda/rope.cu b/src/ggml-cuda/rope.cu -index d332618b..fb5f62bb 100644 +index 33cee75a..ef25d68f 100644 --- a/src/ggml-cuda/rope.cu +++ b/src/ggml-cuda/rope.cu @@ -13,6 +13,9 @@ static __global__ void rope_apply_f32( @@ -1113,10 +1113,10 @@ index d332618b..fb5f62bb 100644 if (index >= pairs) { return; diff --git a/src/ggml.c b/src/ggml.c -index 1fe59a0e..016995c0 100644 +index 1dcc8a0f..ef3c750f 100644 --- a/src/ggml.c +++ b/src/ggml.c -@@ -5751,6 +5751,14 @@ struct ggml_tensor * ggml_rope_pe_permute_rms( +@@ -5773,6 +5773,14 @@ struct ggml_tensor * ggml_rope_pe_permute_rms( return result; } diff --git a/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch b/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch index 2558752fc..b3e8b9a81 100644 --- a/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch +++ b/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch @@ -1,5 +1,5 @@ diff --git a/src/ggml-cpu/arch/x86/cpu-feats.cpp b/src/ggml-cpu/arch/x86/cpu-feats.cpp -index d775a036..e3848609 100644 +index d775a036..a29335b9 100644 --- a/src/ggml-cpu/arch/x86/cpu-feats.cpp +++ b/src/ggml-cpu/arch/x86/cpu-feats.cpp @@ -300,6 +300,10 @@ static int ggml_backend_cpu_x86_score() { diff --git a/scripts/unsloth/ggml-patches/0009-ggml-backend-no-cwd-search.patch b/scripts/unsloth/ggml-patches/0009-ggml-backend-no-cwd-search.patch index 7e20ad043..291d13482 100644 --- a/scripts/unsloth/ggml-patches/0009-ggml-backend-no-cwd-search.patch +++ b/scripts/unsloth/ggml-patches/0009-ggml-backend-no-cwd-search.patch @@ -1,5 +1,5 @@ diff --git a/src/ggml-backend-reg.cpp b/src/ggml-backend-reg.cpp -index 1c18b82..506593d 100644 +index 4e9681c9..a92d21a5 100644 --- a/src/ggml-backend-reg.cpp +++ b/src/ggml-backend-reg.cpp @@ -418,7 +418,8 @@ static fs::path get_executable_path() { @@ -55,7 +55,7 @@ index 1c18b82..506593d 100644 - std::error_code cwd_ec; - const fs::path cwd = fs::current_path(cwd_ec); - if (cwd_ec) { -- GGML_LOG_DEBUG("%s: current_path() failure, error-message: %s\n", __func__, cwd_ec.message().c_str()); +- GGML_LOG_DEBUG("%s: failed to get current directory: %s\n", __func__, cwd_ec.message().c_str()); - } else { - search_paths.push_back(cwd); + // Unsloth: default search path is the executable directory only. Every matching library is diff --git a/scripts/unsloth/ggml-patches/0010-ggml-metal-rope-apply.patch b/scripts/unsloth/ggml-patches/0010-ggml-metal-rope-apply.patch index f1ed13543..280996ed2 100644 --- a/scripts/unsloth/ggml-patches/0010-ggml-metal-rope-apply.patch +++ b/scripts/unsloth/ggml-patches/0010-ggml-metal-rope-apply.patch @@ -1,8 +1,8 @@ diff --git a/src/ggml-metal/ggml-metal-device.cpp b/src/ggml-metal/ggml-metal-device.cpp -index dc6b695e..d1a624ad 100644 +index 2c7972b0..1d2e0318 100644 --- a/src/ggml-metal/ggml-metal-device.cpp +++ b/src/ggml-metal/ggml-metal-device.cpp -@@ -2391,6 +2391,20 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_roll(ggml_metal_ +@@ -2564,6 +2564,20 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_roll(ggml_metal_ return res; } @@ -24,10 +24,10 @@ index dc6b695e..d1a624ad 100644 assert(op->op == GGML_OP_PAD); diff --git a/src/ggml-metal/ggml-metal-device.h b/src/ggml-metal/ggml-metal-device.h -index 1bdaecc7..b41813a9 100644 +index 4012b9d5..48d77221 100644 --- a/src/ggml-metal/ggml-metal-device.h +++ b/src/ggml-metal/ggml-metal-device.h -@@ -171,6 +171,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_upscale +@@ -173,6 +173,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_upscale struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pad (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pad_reflect_1d (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_roll (ggml_metal_library_t lib, const struct ggml_tensor * op); @@ -36,10 +36,10 @@ index 1bdaecc7..b41813a9 100644 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_timestep_embedding(ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_opt_step_adamw (ggml_metal_library_t lib, const struct ggml_tensor * op); diff --git a/src/ggml-metal/ggml-metal-device.m b/src/ggml-metal/ggml-metal-device.m -index a81d3a8c..498ee7ca 100644 +index 44519903..8634d284 100644 --- a/src/ggml-metal/ggml-metal-device.m +++ b/src/ggml-metal/ggml-metal-device.m -@@ -1681,6 +1681,10 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te +@@ -1695,6 +1695,10 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_OP_ROPE: case GGML_OP_ROPE_BACK: return true; @@ -51,10 +51,10 @@ index a81d3a8c..498ee7ca 100644 return ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32 && (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32); case GGML_OP_CONV_2D: diff --git a/src/ggml-metal/ggml-metal-impl.h b/src/ggml-metal/ggml-metal-impl.h -index 490dd83a..d61e0fcb 100644 +index 4ffb5a77..a358e659 100644 --- a/src/ggml-metal/ggml-metal-impl.h +++ b/src/ggml-metal/ggml-metal-impl.h -@@ -1166,6 +1166,22 @@ typedef struct { +@@ -1177,6 +1177,22 @@ typedef struct { int32_t s3; } ggml_metal_kargs_roll; @@ -78,10 +78,10 @@ index 490dd83a..d61e0fcb 100644 uint64_t nb1; int dim; diff --git a/src/ggml-metal/ggml-metal-ops.cpp b/src/ggml-metal/ggml-metal-ops.cpp -index 708703e5..031adb8a 100644 +index 53c98161..551d62b0 100644 --- a/src/ggml-metal/ggml-metal-ops.cpp +++ b/src/ggml-metal/ggml-metal-ops.cpp -@@ -410,6 +410,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { +@@ -414,6 +414,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { { n_fuse = ggml_metal_op_rope(ctx, idx); } break; @@ -92,7 +92,7 @@ index 708703e5..031adb8a 100644 case GGML_OP_IM2COL: { n_fuse = ggml_metal_op_im2col(ctx, idx); -@@ -4977,6 +4981,47 @@ int ggml_metal_op_roll(ggml_metal_op_t ctx, int idx) { +@@ -5257,6 +5261,47 @@ int ggml_metal_op_roll(ggml_metal_op_t ctx, int idx) { return 1; } diff --git a/scripts/unsloth/ggml-patches/0011-ggml-metal-no-fp8-int8-mul-mat.patch b/scripts/unsloth/ggml-patches/0011-ggml-metal-no-fp8-int8-mul-mat.patch index 69eb343e5..0b44b86fe 100644 --- a/scripts/unsloth/ggml-patches/0011-ggml-metal-no-fp8-int8-mul-mat.patch +++ b/scripts/unsloth/ggml-patches/0011-ggml-metal-no-fp8-int8-mul-mat.patch @@ -1,8 +1,8 @@ diff --git a/src/ggml-metal/ggml-metal-device.m b/src/ggml-metal/ggml-metal-device.m -index 498ee7ca..80aaf0ec 100644 +index 8634d284..1dc0d248 100644 --- a/src/ggml-metal/ggml-metal-device.m +++ b/src/ggml-metal/ggml-metal-device.m -@@ -1507,6 +1507,13 @@ static bool ggml_metal_supports_mul_mat_op( +@@ -1521,6 +1521,13 @@ static bool ggml_metal_supports_mul_mat_op( return false; } diff --git a/scripts/unsloth/ggml-patches/0012-ggml-metal-group-norm-threads.patch b/scripts/unsloth/ggml-patches/0012-ggml-metal-group-norm-threads.patch index 2d7051b70..b65cbdddd 100644 --- a/scripts/unsloth/ggml-patches/0012-ggml-metal-group-norm-threads.patch +++ b/scripts/unsloth/ggml-patches/0012-ggml-metal-group-norm-threads.patch @@ -1,8 +1,8 @@ diff --git a/src/ggml-metal/ggml-metal-ops.cpp b/src/ggml-metal/ggml-metal-ops.cpp -index 031adb8a..aacf12c0 100644 +index 551d62b0..1aa237bc 100644 --- a/src/ggml-metal/ggml-metal-ops.cpp +++ b/src/ggml-metal/ggml-metal-ops.cpp -@@ -4103,13 +4103,13 @@ int ggml_metal_op_group_norm(ggml_metal_op_t ctx, int idx) { +@@ -4383,13 +4383,13 @@ int ggml_metal_op_group_norm(ggml_metal_op_t ctx, int idx) { auto pipeline = ggml_metal_library_get_pipeline_group_norm(lib, op); @@ -22,7 +22,7 @@ index 031adb8a..aacf12c0 100644 const size_t smem = pipeline.smem; -@@ -4120,7 +4120,7 @@ int ggml_metal_op_group_norm(ggml_metal_op_t ctx, int idx) { +@@ -4400,7 +4400,7 @@ int ggml_metal_op_group_norm(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); diff --git a/scripts/unsloth/ggml-patches/0013-ggml-sched-inplace-op-owner-backend.patch b/scripts/unsloth/ggml-patches/0013-ggml-sched-inplace-op-owner-backend.patch index c3d3b8579..fe8b11351 100644 --- a/scripts/unsloth/ggml-patches/0013-ggml-sched-inplace-op-owner-backend.patch +++ b/scripts/unsloth/ggml-patches/0013-ggml-sched-inplace-op-owner-backend.patch @@ -1,8 +1,8 @@ diff --git a/src/ggml-backend.cpp b/src/ggml-backend.cpp -index 8c348734..cc597430 100644 +index 48802d68..a1cfe7da 100644 --- a/src/ggml-backend.cpp +++ b/src/ggml-backend.cpp -@@ -1294,6 +1294,27 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra +@@ -1442,6 +1442,27 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra GGML_ASSERT(*cur_backend_id != -1); } diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index d5fa6750d..7bbc38348 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -1995,6 +1995,9 @@ struct LLMEmbedder : public Conditioner { if (sd_version_is_pid(version)) { throw std::runtime_error("PiD requires an external Gemma 2 tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer"); } + if (version == VERSION_MOSS_IMAGE) { + throw std::runtime_error("MOSS-Image requires the official Qwen3.5 tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer"); + } } LLM::LLMArch arch = LLM::LLMArch::QWEN2_5_VL; if (version == VERSION_FLUX2) { @@ -2018,6 +2021,8 @@ struct LLMEmbedder : public Conditioner { arch = LLM::LLMArch::QWEN3_VL; } else if (sd_version_is_z_image(version) || sd_version_is_z_image_l2p(version) || version == VERSION_OVIS_IMAGE || version == VERSION_FLUX2_KLEIN) { arch = LLM::LLMArch::QWEN3; + } else if (version == VERSION_MOSS_IMAGE) { + arch = LLM::LLMArch::QWEN3_5; } llm = std::make_shared(arch, backend, @@ -2032,6 +2037,8 @@ struct LLMEmbedder : public Conditioner { pad_id = 199999; } else if (arch == LLM::LLMArch::GEMMA2_2B) { pad_id = 0; + } else if (arch == LLM::LLMArch::QWEN3_5) { + pad_id = 248044; } tokenizer = tokenizers.create(TokenizerConfig::MAIN, llm->config.vocab_size, pad_id); if (!tokenizer) { @@ -3039,6 +3046,11 @@ struct LLMEmbedder : public Conditioner { prompt += "<|im_end|>\n<|im_start|>assistant\n"; } + } else if (version == VERSION_MOSS_IMAGE) { + prompt_template_encode_start_idx = 0; + max_length = 4096; + prompt = "<|im_start|>user\n" + conditioner_params.text + "<|im_end|>\n<|im_start|>assistant\n\n"; + prompt_attn_range = {0, 0}; } else if (sd_version_is_flux3(version)) { prompt_template_encode_start_idx = 0; max_length = 4096; diff --git a/src/convert.cpp b/src/convert.cpp index 7fb0825f5..b8a62121d 100644 --- a/src/convert.cpp +++ b/src/convert.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -108,7 +109,7 @@ static bool preallocate_output_file(const std::string& output_path, uint64_t fil return true; } - std::fstream file(output_path, std::ios::binary | std::ios::in | std::ios::out); + std::fstream file(std::filesystem::u8path(output_path), std::ios::binary | std::ios::in | std::ios::out); if (!file.is_open()) { if (error != nullptr) { *error = "failed to open output file '" + output_path + "' for preallocation"; @@ -251,7 +252,7 @@ static bool stream_tensor_data(ModelLoader& model_loader, for (int worker = 0; worker < n_threads; worker++) { workers.emplace_back([&]() { - std::fstream output_file(output_path, std::ios::binary | std::ios::in | std::ios::out); + std::fstream output_file(std::filesystem::u8path(output_path), std::ios::binary | std::ios::in | std::ios::out); if (!output_file.is_open()) { fail("failed to open output file '" + output_path + "' for tensor writing"); return; diff --git a/src/core/ggml_extend.cpp b/src/core/ggml_extend.cpp index 3e52b90a0..80e2121c6 100644 --- a/src/core/ggml_extend.cpp +++ b/src/core/ggml_extend.cpp @@ -371,22 +371,15 @@ ggml_tensor* ggml_ext_pad(ggml_context* ctx, } static ggml_tensor* conv_1d(ggml_context* ctx, ggml_tensor* x, ggml_tensor* w, int s0, int p0, int d0, bool force_prec_f32) { - ggml_tensor* result; - if (force_prec_f32) { - ggml_tensor* patches = ggml_im2col(ctx, w, x, s0, 0, p0, 0, d0, 0, false, GGML_TYPE_F32); - result = ggml_mul_mat(ctx, - ggml_reshape_2d(ctx, patches, patches->ne[0], patches->ne[2] * patches->ne[1]), - ggml_reshape_2d(ctx, w, w->ne[0] * w->ne[1], w->ne[2])); - result = ggml_reshape_3d(ctx, result, patches->ne[1], w->ne[2], patches->ne[2]); - } else { - result = ggml_conv_1d(ctx, w, x, s0, p0, d0); - } - if (x->ne[2] > 1) { - // mul_mat packs positions and batches before output channels: [OL, N, OC]. - result = ggml_reshape_3d(ctx, result, result->ne[0], x->ne[2], w->ne[2]); - result = ggml_cont(ctx, ggml_permute(ctx, result, 0, 2, 1, 3)); - } - return result; + if (!force_prec_f32) { + return ggml_conv_1d(ctx, w, x, s0, p0, d0); + } + ggml_tensor* patches = ggml_im2col(ctx, w, x, s0, 0, p0, 0, d0, 0, false, GGML_TYPE_F32); + ggml_tensor* result = ggml_mul_mat(ctx, + ggml_reshape_2d(ctx, patches, patches->ne[0], patches->ne[2] * patches->ne[1]), + ggml_reshape_2d(ctx, w, w->ne[0] * w->ne[1], w->ne[2])); + result = ggml_reshape_3d(ctx, result, patches->ne[1], patches->ne[2], w->ne[2]); + return ggml_cont(ctx, ggml_permute(ctx, result, 0, 2, 1, 3)); } ggml_tensor* ggml_ext_conv_1d(ggml_context* ctx, diff --git a/src/core/ggml_tensor_utils.cpp b/src/core/ggml_tensor_utils.cpp index 1747678d4..5639c7ed9 100644 --- a/src/core/ggml_tensor_utils.cpp +++ b/src/core/ggml_tensor_utils.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include "core/ggml_extend_backend.h" @@ -100,7 +101,7 @@ void ggml_ext_tensor_diff( } ggml_tensor* load_tensor_from_file(ggml_context* ctx, const std::string& file_path) { - std::ifstream file(file_path, std::ios::binary); + std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary); if (!file.is_open()) { LOG_ERROR("failed to open '%s'", file_path.c_str()); return nullptr; diff --git a/src/core/tensor_ggml.hpp b/src/core/tensor_ggml.hpp index 66ea22206..5082ce207 100644 --- a/src/core/tensor_ggml.hpp +++ b/src/core/tensor_ggml.hpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -105,7 +106,7 @@ namespace sd { template inline Tensor load_tensor_from_file_as_tensor(const std::string& file_path) { - std::ifstream file(file_path, std::ios::binary); + std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary); if (!file.is_open()) { throw std::runtime_error("failed to open tensor file: " + file_path); } diff --git a/src/core/util.cpp b/src/core/util.cpp index 5292c38ba..c1d463e4d 100644 --- a/src/core/util.cpp +++ b/src/core/util.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -123,12 +124,12 @@ int round_up_to(int value, int base) { #include bool file_exists(const std::string& filename) { - DWORD attributes = GetFileAttributesA(filename.c_str()); + DWORD attributes = GetFileAttributesW(std::filesystem::u8path(filename).c_str()); return (attributes != INVALID_FILE_ATTRIBUTES && !(attributes & FILE_ATTRIBUTE_DIRECTORY)); } bool is_directory(const std::string& path) { - DWORD attributes = GetFileAttributesA(path.c_str()); + DWORD attributes = GetFileAttributesW(std::filesystem::u8path(path).c_str()); return (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY)); } @@ -152,8 +153,8 @@ std::unique_ptr MmapWrapper::create(const std::string& filename, bo void* mapped_data = nullptr; size_t file_size = 0; - HANDLE file_handle = CreateFileA( - filename.c_str(), + HANDLE file_handle = CreateFileW( + std::filesystem::u8path(filename).c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, diff --git a/src/model.h b/src/model.h index baf3a2a34..0c1a37a66 100644 --- a/src/model.h +++ b/src/model.h @@ -67,6 +67,7 @@ enum SDVersion { VERSION_Z_IMAGE_L2P, VERSION_IRIS, VERSION_FLUX3, + VERSION_MOSS_IMAGE, VERSION_COUNT, }; @@ -291,7 +292,8 @@ static inline bool sd_version_uses_flux2_vae(SDVersion version) { sd_version_is_lens(version) || sd_version_is_ideogram4(version) || sd_version_is_sefi_image(version) || - sd_version_is_llada_image(version)) { + sd_version_is_llada_image(version) || + version == VERSION_MOSS_IMAGE) { return true; } return false; @@ -348,7 +350,8 @@ static inline bool sd_version_is_dit(SDVersion version) { sd_version_is_mage_flow(version) || sd_version_is_sensenova_u1(version) || sd_version_is_pixart(version) || - version == VERSION_IRIS) { + version == VERSION_IRIS || + version == VERSION_MOSS_IMAGE) { return true; } return false; @@ -362,6 +365,29 @@ static inline bool sd_version_is_control(SDVersion version) { return version == VERSION_FLUX_CONTROLS || version == VERSION_FLEX_2; } +// These model families consume uploaded reference images as generation +// conditioning. Inpaint and control variants keep their init-image semantics. +static inline bool sd_version_supports_ref_image_conditioning(SDVersion version) { + if (sd_version_is_inpaint(version) || sd_version_is_control(version) || version == VERSION_CHROMA_RADIANCE) { + return false; + } + + return sd_version_is_unet_edit(version) || + sd_version_is_flux(version) || + sd_version_is_flux3(version) || + sd_version_is_flux2(version) || + sd_version_is_qwen_image(version) || + sd_version_is_longcat(version) || + sd_version_is_z_image(version) || + sd_version_is_boogu_image(version) || + sd_version_is_llada_image(version) || + sd_version_is_krea2(version) || + sd_version_is_mage_flow(version) || + sd_version_is_anima(version) || + sd_version_is_sefi_image(version) || + sd_version_is_pid(version); +} + static bool sd_version_is_inpaint_or_unet_edit(SDVersion version) { return sd_version_is_unet_edit(version) || sd_version_is_inpaint(version) || sd_version_is_control(version); } diff --git a/src/model/adapter/lora.hpp b/src/model/adapter/lora.hpp index e9d20a4ec..1e5377ccc 100644 --- a/src/model/adapter/lora.hpp +++ b/src/model/adapter/lora.hpp @@ -1,6 +1,7 @@ #ifndef __SD_MODEL_ADAPTER_LORA_HPP__ #define __SD_MODEL_ADAPTER_LORA_HPP__ +#include #include #include "core/ggml_extend.h" #include "core/ggml_extend_backend.h" @@ -11,15 +12,18 @@ #include "model/adapter/lora_ops.h" #include "model_loader.h" #include "model_manager.h" +#include "name_conversion.h" #define LORA_GRAPH_BASE_SIZE 10240 struct LoraModel : public GGMLRunner { std::string lora_id; - float multiplier = 1.0f; + float multiplier = 1.0f; + float metadata_alpha = -1.f; std::unordered_map lora_tensors; std::map original_tensor_to_final_tensor; std::set applied_lora_tensors; + std::set swapped_lora_up_tensors; std::set skipped_incompatible_lora_tensors; std::set warned_incompatible_model_tensors; std::string file_path; @@ -62,6 +66,31 @@ struct LoraModel : public GGMLRunner { if (load_failed || !registered_params.empty()) return false; model_manager->set_n_threads(n_threads); + metadata_alpha = -1.f; + const auto metadata = model_manager->loader().file_metadata(source_file); + const auto alpha = metadata.find("alpha"); + if (alpha != metadata.end()) { + try { + size_t parsed = 0; + metadata_alpha = std::stof(alpha->second, &parsed); + if (parsed != alpha->second.size() || !std::isfinite(metadata_alpha) || metadata_alpha < 0.f) { + throw std::invalid_argument("invalid alpha"); + } + } catch (const std::exception&) { + LOG_ERROR("invalid LoRA alpha metadata '%s' in '%s'", alpha->second.c_str(), file_path.c_str()); + return false; + } + LOG_VERBOSE("LoRA '%s' uses metadata alpha %.6g when tensor alpha/scale is absent", file_path.c_str(), metadata_alpha); + } + if (sd_version_is_minimax_h3(source_version)) { + for (const auto& entry : model_manager->loader().file_tensors(source_file)) { + const auto name = convert_tensor_name(entry.first, VERSION_COUNT); + if (ends_with(name, ".ff.net.0.proj.weight.lora_up")) { + // Diffusers SwiGLU stores [value, gate]; MiniMax-H3 uses [gate, value]. + swapped_lora_up_tensors.insert(convert_tensor_name(entry.first, source_version)); + } + } + } const auto sources = model_manager->loader().file_tensors(source_file, source_version); std::map tensors; std::map scalars; @@ -116,6 +145,8 @@ struct LoraModel : public GGMLRunner { scalar_values.clear(); original_tensor_to_final_tensor.clear(); applied_lora_tensors.clear(); + swapped_lora_up_tensors.clear(); + metadata_alpha = -1.f; skipped_incompatible_lora_tensors.clear(); warned_incompatible_model_tensors.clear(); applied = false; @@ -210,6 +241,10 @@ struct LoraModel : public GGMLRunner { auto iter = lora_tensors.find(lora_up_name); if (iter != lora_tensors.end()) { lora_up = ggml_ext_cast_f32(ctx, backend, iter->second); + if (swapped_lora_up_tensors.count(lora_up_name)) { + auto halves = ggml_ext_chunk(ctx, lora_up, 2, 1); + lora_up = ggml_concat(ctx, halves[1], halves[0], 1); + } } iter = lora_tensors.find(lora_mid_name); @@ -236,6 +271,7 @@ struct LoraModel : public GGMLRunner { float scale_value = 1.0f; int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1]; + scale_value = metadata_alpha >= 0.f ? metadata_alpha / rank : 1.f; iter = lora_tensors.find(scale_name); if (iter != lora_tensors.end()) { scale_value = scalar_value(iter->second); @@ -723,6 +759,10 @@ struct LoraModel : public GGMLRunner { iter = lora_tensors.find(lora_up_name); if (iter != lora_tensors.end()) { lora_up = iter->second; + if (swapped_lora_up_tensors.count(lora_up_name)) { + auto halves = ggml_ext_chunk(ctx, lora_up, 2, 1); + lora_up = ggml_concat(ctx, halves[1], halves[0], 1); + } if (is_conv2d && lora_up->type != GGML_TYPE_F16) { lora_up = ggml_cast(ctx, lora_up, GGML_TYPE_F16); } @@ -791,6 +831,7 @@ struct LoraModel : public GGMLRunner { std::string scale_tensor_name; int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1]; + scale_value = metadata_alpha >= 0.f ? metadata_alpha / rank : 1.f; iter = lora_tensors.find(scale_name); if (iter != lora_tensors.end()) { scale_value = scalar_value(iter->second); diff --git a/src/model/common/ggml_block.hpp b/src/model/common/ggml_block.hpp index cf2e90fb5..be6dfea3d 100644 --- a/src/model/common/ggml_block.hpp +++ b/src/model/common/ggml_block.hpp @@ -15,7 +15,6 @@ #include "core/ggml_extend.h" #include "core/ggml_extend_backend.h" #include "core/ggml_runner.h" -#include "core/ggml_extend_backend.h" #include "model.h" class GGMLBlock { diff --git a/src/model/diffusion/moss_image.hpp b/src/model/diffusion/moss_image.hpp new file mode 100644 index 000000000..f1cbbc7f3 --- /dev/null +++ b/src/model/diffusion/moss_image.hpp @@ -0,0 +1,262 @@ +#ifndef __SD_MODEL_DIFFUSION_MOSS_IMAGE_HPP__ +#define __SD_MODEL_DIFFUSION_MOSS_IMAGE_HPP__ + +#include +#include + +#include "model/common/ggml_block.hpp" +#include "model/common/rope.hpp" +#include "model/diffusion/dit.hpp" +#include "model/diffusion/model.hpp" +#include "model_loader.h" + +namespace MOSSImage { + struct MOSSImageConfig { + int64_t dim = 3072; + int64_t ffn_dim = 8192; + int64_t text_dim = 4096; + int64_t channels = 128; + int64_t head_dim = 128; + int64_t depth = 42; + int64_t freq_dim = 256; + + static MOSSImageConfig detect_from_weights(const String2TensorStorage& weights, const std::string& prefix) { + MOSSImageConfig config; + int64_t depth = 0; + for (const auto& [name, tensor] : weights) { + if (!starts_with(name, prefix + ".")) { + continue; + } + auto key = name.substr(prefix.size() + 1); + if (key == "video_embedder.weight") { + config.channels = tensor.ne[0]; + config.dim = tensor.ne[1]; + } else if (key == "text_embedder.proj.weight") { + config.text_dim = tensor.ne[0]; + } else if (key == "layers.0.proj.ffn_down.weight") { + config.ffn_dim = tensor.ne[0]; + } else if (key == "layers.0.q_norm.weight") { + config.head_dim = tensor.ne[0]; + } else if (key == "time_embedding.0.weight") { + config.freq_dim = tensor.ne[0]; + } + if (starts_with(key, "layers.")) { + depth = std::max(depth, std::stoi(key.substr(7)) + 1); + } + } + if (depth) { + config.depth = depth; + LOG_VERBOSE("moss_image: dim = %" PRId64 ", ffn_dim = %" PRId64 ", depth = %" PRId64 ", head_dim = %" PRId64, + config.dim, config.ffn_dim, config.depth, config.head_dim); + } + return config; + } + }; + + inline ggml_tensor* situ(ggml_context* c, ggml_tensor* hidden, ggml_tensor* gate) { + auto a = ggml_scale(c, ggml_tanh(c, ggml_scale(c, gate, 0.25f)), 4.f); + a = ggml_mul(c, a, ggml_sigmoid(c, gate)); + auto b = ggml_scale(c, ggml_tanh(c, ggml_scale(c, hidden, 1.f / 25.f)), 25.f); + return ggml_mul(c, a, b); + } + + struct TextBlock : public GGMLBlock { + MOSSImageConfig config; + + TextBlock(const MOSSImageConfig& config) + : config(config) { + blocks["norm1"] = std::make_shared(config.dim, 1e-6f); + blocks["norm2"] = std::make_shared(config.dim, 1e-6f); + blocks["q_norm"] = std::make_shared(config.head_dim, 1e-6f); + blocks["k_norm"] = std::make_shared(config.head_dim, 1e-6f); + blocks["qkv"] = std::make_shared(config.dim, config.dim * 3, true); + blocks["proj"] = std::make_shared(config.dim, config.dim, true); + blocks["ffn_up"] = std::make_shared(config.dim, config.ffn_dim, true); + blocks["ffn_gate"] = std::make_shared(config.dim, config.ffn_dim, true); + blocks["ffn_down"] = std::make_shared(config.ffn_dim, config.dim, true); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe) { + auto c = ctx->ggml_ctx; + auto linear = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast(blocks[name])->forward(ctx, h); }; + auto norm = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast(blocks[name])->forward(ctx, h); }; + auto y = norm("norm1", x); + auto qkv = linear("qkv", y); + auto split = [&](int i) { + return ggml_reshape_4d(c, ggml_ext_slice(c, qkv, 0, i * config.dim, (i + 1) * config.dim), config.head_dim, config.dim / config.head_dim, x->ne[1], x->ne[2]); + }; + auto q = norm("q_norm", split(0)); + auto k = norm("k_norm", split(1)); + auto v = split(2); + y = Rope::attention(ctx, q, k, v, pe, nullptr, 1.f, false); + y = linear("proj", y); + x = ggml_add(c, x, y); + y = norm("norm2", x); + y = situ(c, linear("ffn_up", y), linear("ffn_gate", y)); + y = linear("ffn_down", y); + return ggml_add(c, x, y); + } + }; + + struct Block : public GGMLBlock { + MOSSImageConfig config; + + Block(const MOSSImageConfig& config) + : config(config) { + blocks["norm1"] = std::make_shared(config.dim, 1e-6f); + blocks["norm2"] = std::make_shared(config.dim, 1e-6f); + blocks["q_norm"] = std::make_shared(config.head_dim, 1e-6f); + blocks["k_norm"] = std::make_shared(config.head_dim, 1e-6f); + blocks["proj.qkv"] = std::make_shared(config.dim, config.dim * 3, false); + blocks["proj.o"] = std::make_shared(config.dim, config.dim, false); + blocks["proj.ffn_up_gate"] = std::make_shared(config.dim, config.ffn_dim * 2, false); + blocks["proj.ffn_down"] = std::make_shared(config.ffn_dim, config.dim, false); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe, ggml_tensor* mod) { + auto c = ctx->ggml_ctx; + auto linear = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast(blocks[name])->forward(ctx, h); }; + auto norm = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast(blocks[name])->forward(ctx, h); }; + auto part = [&](int i) { return ggml_ext_slice(c, mod, 0, i * config.dim, (i + 1) * config.dim); }; + auto y = norm("norm1", x); + y = ggml_add(c, ggml_add(c, y, ggml_mul(c, y, part(1))), part(0)); + auto qkv = linear("proj.qkv", y); + auto split = [&](int i) { + return ggml_reshape_4d(c, ggml_ext_slice(c, qkv, 0, i * config.dim, (i + 1) * config.dim), config.head_dim, config.dim / config.head_dim, x->ne[1], x->ne[2]); + }; + auto q = norm("q_norm", split(0)); + auto k = norm("k_norm", split(1)); + auto v = split(2); + y = Rope::attention(ctx, q, k, v, pe, nullptr, 1.f, false); + y = linear("proj.o", y); + y = ggml_mul(c, y, part(2)); + x = ggml_add(c, x, y); + y = norm("norm2", x); + y = ggml_add(c, ggml_add(c, y, ggml_mul(c, y, part(4))), part(3)); + auto up_gate = linear("proj.ffn_up_gate", y); + y = situ(c, ggml_ext_slice(c, up_gate, 0, 0, config.ffn_dim), ggml_ext_slice(c, up_gate, 0, config.ffn_dim, 2 * config.ffn_dim)); + y = linear("proj.ffn_down", y); + y = ggml_mul(c, y, part(5)); + return ggml_add(c, x, y); + } + }; + + struct Model : public GGMLBlock { + MOSSImageConfig config; + + Model(const MOSSImageConfig& config) + : config(config) { + blocks["video_embedder"] = std::make_shared(config.channels, config.dim, true); + blocks["text_embedder.proj"] = std::make_shared(config.text_dim, config.dim, true); + for (int i = 0; i < 2; ++i) { + blocks["text_embedder.blocks." + std::to_string(i)] = std::make_shared(config); + } + blocks["time_embedding.0"] = std::make_shared(config.freq_dim, config.dim, true); + blocks["time_embedding.2"] = std::make_shared(config.dim, config.dim, true); + blocks["shared_block_modulation"] = std::make_shared(config.dim, 6 * config.dim, true); + blocks["video_head_modulation"] = std::make_shared(config.dim, 2 * config.dim, true); + for (int64_t i = 0; i < config.depth; ++i) { + blocks["layers." + std::to_string(i)] = std::make_shared(config); + } + blocks["final_norm_video"] = std::make_shared(config.dim, 1e-6f); + blocks["final_linear_video"] = std::make_shared(config.dim, config.channels, false); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* image, ggml_tensor* time, ggml_tensor* text, ggml_tensor* pe, const std::vector& skip) { + auto c = ctx->ggml_ctx; + auto linear = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast(blocks[name])->forward(ctx, h); }; + int64_t image_len = image->ne[0] * image->ne[1]; + auto x = linear("video_embedder", DiT::patchify(c, image, 1, 1)); + text = linear("text_embedder.proj", text); + auto text_pe = ggml_ext_slice(c, pe, 3, image_len, image_len + text->ne[1]); + for (int i = 0; i < 2; ++i) { + text = std::dynamic_pointer_cast(blocks["text_embedder.blocks." + std::to_string(i)])->forward(ctx, text, text_pe); + } + time = linear("time_embedding.2", ggml_silu(c, linear("time_embedding.0", time))); + auto mod = linear("shared_block_modulation", time); + auto head = linear("video_head_modulation", time); + x = ggml_concat(c, x, text, 1); + for (int64_t i = 0; i < config.depth; ++i) { + if (std::find(skip.begin(), skip.end(), i) == skip.end()) { + x = std::dynamic_pointer_cast(blocks["layers." + std::to_string(i)])->forward(ctx, x, pe, mod); + sd::ggml_graph_cut::mark_graph_cut(x, "moss_image.layers." + std::to_string(i), "x"); + } + } + x = ggml_ext_slice(c, x, 1, 0, image_len); + x = std::dynamic_pointer_cast(blocks["final_norm_video"])->forward(ctx, x); + auto shift = ggml_ext_slice(c, head, 0, 0, config.dim); + auto scale = ggml_ext_slice(c, head, 0, config.dim, 2 * config.dim); + x = ggml_add(c, ggml_add(c, x, ggml_mul(c, x, scale)), shift); + x = linear("final_linear_video", x); + return DiT::unpatchify(c, x, image->ne[1], image->ne[0], 1, 1); + } + }; + + struct MOSSImageRunner : public DiffusionModelRunner { + MOSSImageConfig config; + Model model; + std::vector pe_data, time_data; + + MOSSImageRunner(ggml_backend_t backend, const String2TensorStorage& weights, const std::string& prefix, std::shared_ptr weight_manager = nullptr) + : DiffusionModelRunner(backend, prefix, weight_manager), config(MOSSImageConfig::detect_from_weights(weights, prefix)), model(config) { + model.init(params_ctx, weights, prefix); + } + + std::string get_desc() override { return "MOSS-Image"; } + void get_param_tensors(std::map& tensors, const std::string& prefix) override { + model.get_param_tensors(tensors, prefix); + } + + sd::Tensor compute(int n_threads, const DiffusionParams& inputs) override { + if (!inputs.x || !inputs.context || !inputs.timesteps || inputs.context->empty()) { + LOG_ERROR("MOSS-Image requires latents, timesteps and text features"); + return {}; + } + std::vector skip; + if (auto extra = std::get_if(&inputs.extra)) { + if (extra->skip_layers) { + skip = *extra->skip_layers; + } + } + auto graph = [&]() { + auto gf = new_graph_custom(65536); + auto x = make_input(*inputs.x); + auto text = make_input(*inputs.context); + GGML_ASSERT(x->ne[3] == 1 && x->ne[2] == config.channels && text->ne[0] == config.text_dim); + int64_t h = x->ne[1], w = x->ne[0], nt = text->ne[1]; + Rope::Embedding embedding; + embedding.positions.append_image(static_cast(h), static_cast(w)); + embedding.positions.images.back().height_axis = 2; + embedding.positions.images.back().width_axis = 3; + embedding.positions.append_tokens(nt); + for (int64_t token = 0; token < h * w + nt; ++token) { + float pos[4] = {0.f, float(token - h * w), 0.f, 0.f}; + if (token < h * w) { + pos[1] = float(nt); + pos[2] = float(token / w) - float(h - 1) / 2.f; + pos[3] = float(token % w) - float(w - 1) / 2.f; + } + embedding.ids.push_back({pos[0], pos[1], pos[2], pos[3]}); + } + embedding.values = Rope::embed_nd(embedding.ids, 1, 10000.f, std::vector(4, static_cast(config.head_dim / 4)), embedding.layout, &embedding.frequencies); + pe_data = finish_rope_pe(std::move(embedding)); + time_data.resize(config.freq_dim); + for (int64_t i = 0; i < config.freq_dim / 2; ++i) { + double angle = inputs.timesteps->values()[0] * std::pow(10000.0, -double(i) / double(config.freq_dim / 2)); + time_data[i] = float(std::cos(angle)); + time_data[i + config.freq_dim / 2] = float(std::sin(angle)); + } + auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.head_dim / 2, h * w + nt); + auto time = ggml_new_tensor_3d(compute_ctx, GGML_TYPE_F32, config.freq_dim, 1, 1); + set_backend_tensor_data(pe, pe_data.data()); + set_backend_tensor_data(time, time_data.data()); + auto ctx = get_context(); + ggml_build_forward_expand(gf, model.forward(&ctx, x, time, text, pe, skip)); + return gf; + }; + return restore_trailing_singleton_dims(GGMLRunner::compute(graph, n_threads, false), inputs.x->dim()); + } + }; +} // namespace MOSSImage + +#endif // __SD_MODEL_DIFFUSION_MOSS_IMAGE_HPP__ diff --git a/src/model/te/llm.hpp b/src/model/te/llm.hpp index a782ea7bd..8ea918025 100644 --- a/src/model/te/llm.hpp +++ b/src/model/te/llm.hpp @@ -43,6 +43,7 @@ namespace LLM { QWEN2_5_VL, QWEN3, QWEN3_VL, + QWEN3_5, MISTRAL_SMALL_3_2, MINISTRAL_3_3B, GEMMA3_12B, @@ -59,6 +60,7 @@ namespace LLM { "qwen2.5vl", "qwen3", "qwen3vl", + "qwen3.5", "mistral_small3.2", "ministral3.3b", "gemma3_12b", @@ -121,6 +123,7 @@ namespace LLM { bool qk_norm = false; bool final_norm = true; bool rms_norm_add = false; + bool qwen35_gguf = false; bool normalize_input = false; int64_t vocab_size = 152064; int64_t max_position_embeddings = 128000; @@ -134,6 +137,11 @@ namespace LLM { bool qkv_fused = false; bool bidirectional = false; float partial_rotary = 1.f; + int linear_head_dim = 128; + int linear_key_heads = 16; + int linear_value_heads = 32; + int linear_conv_kernel = 4; + std::set linear_attention_layers; // DeepSeek-V3-style grouped-sigmoid MoE routing (LLaDA2) int64_t moe_intermediate_size = 0; @@ -167,6 +175,19 @@ namespace LLM { config.num_kv_heads = 8; config.qkv_bias = false; config.rms_norm_eps = 1e-5f; + } else if (arch == LLMArch::QWEN3_5) { + config.hidden_size = 4096; + config.intermediate_size = 12288; + config.head_dim = 256; + config.num_heads = 16; + config.num_kv_heads = 4; + config.qkv_bias = false; + config.qk_norm = true; + config.rms_norm_add = true; + config.partial_rotary = 0.25f; + config.rope_thetas = {10000000.f}; + config.max_position_embeddings = 262144; + config.vocab_size = 248320; } else if (arch == LLMArch::QWEN3 || arch == LLMArch::QWEN3_VL) { config.head_dim = 128; config.num_heads = 32; @@ -365,6 +386,41 @@ namespace LLM { if ((arch == LLMArch::QWEN3 || arch == LLMArch::QWEN3_VL) && config.num_layers == 28) { config.num_heads = 16; } + if (arch == LLMArch::QWEN3_5) { + config.qwen35_gguf = tensor_storage_map.count(prefix + ".model.layers.0.linear_attn.A") > 0; + config.rms_norm_add = !config.qwen35_gguf; + for (const auto& [name, tensor] : tensor_storage_map) { + if (!starts_with(name, prefix + ".")) { + continue; + } + if (ends_with(name, "self_attn.q_norm.weight")) { + config.head_dim = static_cast(tensor.ne[0]); + } else if (ends_with(name, "linear_attn.norm.weight")) { + config.linear_head_dim = static_cast(tensor.ne[0]); + } else if (ends_with(name, "linear_attn.conv1d.weight")) { + config.linear_conv_kernel = static_cast(tensor.ne[0]); + const auto pos = name.find("layers."); + config.linear_attention_layers.insert(std::stoi(name.substr(pos + 7))); + } + } + for (const auto& [name, tensor] : tensor_storage_map) { + if (!starts_with(name, prefix + ".")) { + continue; + } + if (ends_with(name, "self_attn.q_proj.weight")) { + config.num_heads = static_cast(tensor.ne[1] / (2 * config.head_dim)); + } else if (ends_with(name, "self_attn.k_proj.weight")) { + config.num_kv_heads = static_cast(tensor.ne[1] / config.head_dim); + } else if (ends_with(name, "linear_attn.in_proj_z.weight")) { + config.linear_value_heads = static_cast(tensor.ne[1] / config.linear_head_dim); + } + } + for (const auto& [name, tensor] : tensor_storage_map) { + if (starts_with(name, prefix + ".") && ends_with(name, "linear_attn.in_proj_qkv.weight")) { + config.linear_key_heads = static_cast((tensor.ne[1] / config.linear_head_dim - config.linear_value_heads) / 2); + } + } + } if (arch == LLMArch::QWEN3_VL && (config.num_layers == 50 || config.num_layers == 64) && config.hidden_size == 5120) { @@ -1390,6 +1446,81 @@ namespace LLM { } }; + struct Qwen35DeltaNet : public UnaryBlock { + int head_dim; + int key_heads; + int value_heads; + int conv_dim; + int kernel; + bool gguf_layout; + + Qwen35DeltaNet(const LLMConfig& config) + : head_dim(config.linear_head_dim), key_heads(config.linear_key_heads), value_heads(config.linear_value_heads), conv_dim(head_dim * (key_heads * 2 + value_heads)), kernel(config.linear_conv_kernel), gguf_layout(config.qwen35_gguf) { + auto hidden_size = config.hidden_size; + blocks["in_proj_qkv"] = std::make_shared(hidden_size, conv_dim, false); + blocks["in_proj_z"] = std::make_shared(hidden_size, head_dim * value_heads, false); + blocks["in_proj_a"] = std::make_shared(hidden_size, value_heads, false); + blocks["in_proj_b"] = std::make_shared(hidden_size, value_heads, false); + blocks["out_proj"] = std::make_shared(head_dim * value_heads, hidden_size, false); + blocks["norm"] = std::make_shared(head_dim, 1e-6f); + } + + void init_params(ggml_context* ctx, const String2TensorStorage& = {}, std::string = "") override { + params["conv1d.weight"] = gguf_layout ? ggml_new_tensor_2d(ctx, GGML_TYPE_F32, kernel, conv_dim) : ggml_new_tensor_3d(ctx, GGML_TYPE_F32, kernel, 1, conv_dim); + params[gguf_layout ? "A" : "A_log"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, value_heads); + params["dt_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, value_heads); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override { + auto c = ctx->ggml_ctx; + const int64_t n = x->ne[1]; + GGML_ASSERT(x->ne[2] == 1); + auto linear = [&](const char* name) { return std::dynamic_pointer_cast(blocks[name])->forward(ctx, x); }; + auto qkv = linear("in_proj_qkv"); + auto padded = ggml_pad_ext(c, qkv, 0, 0, kernel - 1, 0, 0, 0, 0, 0); + ggml_tensor* mixed = nullptr; + // Causal depthwise convolution, with zero state for every independent prompt. + for (int i = 0; i < kernel; ++i) { + auto w = ggml_cont(c, ggml_view_2d(c, params["conv1d.weight"], 1, conv_dim, kernel * sizeof(float), i * sizeof(float))); + w = ggml_reshape_1d(c, w, conv_dim); + auto part = ggml_mul(c, ggml_ext_slice(c, padded, 1, i, i + n), w); + mixed = mixed ? ggml_add(c, mixed, part) : part; + } + mixed = ggml_silu(c, mixed); + auto heads = [&](int start, int count) { + auto h = ggml_ext_slice(c, mixed, 0, start, start + head_dim * count); + return ggml_reshape_4d(c, h, head_dim, count, n, 1); + }; + auto repeat_keys = [&](ggml_tensor* h) { + h = ggml_scale(c, ggml_rms_norm(c, h, 1e-6f / head_dim), 1.f / std::sqrt(float(head_dim))); + // llama.cpp reorders V heads and stores negative exp(A_log) and offset norms. + if (gguf_layout) { + return h; + } + // ggml GDN broadcasts heads cyclically; Qwen uses adjacent repeated heads. + h = ggml_reshape_4d(c, h, head_dim, 1, key_heads, n); + auto shape = ggml_new_tensor_4d(c, GGML_TYPE_F32, head_dim, value_heads / key_heads, key_heads, n); + return ggml_reshape_4d(c, ggml_repeat(c, h, shape), head_dim, value_heads, n, 1); + }; + auto q = repeat_keys(heads(0, key_heads)); + auto k = repeat_keys(heads(head_dim * key_heads, key_heads)); + auto v = heads(2 * head_dim * key_heads, value_heads); + auto beta = ggml_reshape_4d(c, ggml_sigmoid(c, linear("in_proj_b")), 1, value_heads, n, 1); + auto decay = ggml_softplus(c, ggml_add(c, linear("in_proj_a"), params["dt_bias"])); + auto a = gguf_layout ? params["A"] : ggml_neg(c, ggml_exp(c, params["A_log"])); + decay = ggml_mul(c, decay, a); + decay = ggml_reshape_4d(c, decay, 1, value_heads, n, 1); + auto state = ggml_ext_zeros(c, head_dim, head_dim, value_heads, 1); + auto out = ggml_gated_delta_net(c, q, k, v, decay, beta, state, 1); + out = ggml_view_3d(c, out, head_dim, value_heads, n, head_dim * sizeof(float), head_dim * value_heads * sizeof(float), 0); + out = std::dynamic_pointer_cast(blocks["norm"])->forward(ctx, out); + auto z = ggml_reshape_3d(c, linear("in_proj_z"), head_dim, value_heads, n); + out = ggml_mul(c, out, ggml_silu(c, z)); + out = ggml_reshape_3d(c, out, head_dim * value_heads, n, 1); + return std::dynamic_pointer_cast(blocks["out_proj"])->forward(ctx, out); + } + }; + struct Attention : public GGMLBlock { protected: LLMArch arch; @@ -1439,7 +1570,7 @@ namespace LLM { GGML_ASSERT(!k_eq_v); blocks["query_key_value"] = std::make_shared(config.hidden_size, (num_heads + num_kv_heads * 2) * head_dim, config.qkv_bias); } else { - blocks["q_proj"] = std::make_shared(config.hidden_size, num_heads * head_dim, config.qkv_bias); + blocks["q_proj"] = std::make_shared(config.hidden_size, num_heads * head_dim * (arch == LLMArch::QWEN3_5 ? 2 : 1), config.qkv_bias); blocks["k_proj"] = std::make_shared(config.hidden_size, num_kv_heads * head_dim, config.qkv_bias); if (!k_eq_v) { blocks["v_proj"] = std::make_shared(config.hidden_size, num_kv_heads * head_dim, config.qkv_bias); @@ -1502,6 +1633,13 @@ namespace LLM { v = k_eq_v ? k : v_proj->forward(ctx, x); // [N, n_token, num_kv_heads*head_dim] } + ggml_tensor* output_gate = nullptr; + if (arch == LLMArch::QWEN3_5) { + q = ggml_reshape_4d(ctx->ggml_ctx, q, head_dim * 2, num_heads, n_token, N); + output_gate = ggml_ext_slice(ctx->ggml_ctx, q, 0, head_dim, head_dim * 2); + output_gate = ggml_reshape_3d(ctx->ggml_ctx, output_gate, head_dim * num_heads, n_token, N); + q = ggml_ext_slice(ctx->ggml_ctx, q, 0, 0, head_dim); + } q = ggml_reshape_4d(ctx->ggml_ctx, q, head_dim, num_heads, n_token, N); // [N, n_token, num_heads, head_dim] k = ggml_reshape_4d(ctx->ggml_ctx, k, head_dim, num_kv_heads, n_token, N); // [N, n_token, num_kv_heads, head_dim] v = ggml_reshape_4d(ctx->ggml_ctx, v, head_dim, num_kv_heads, n_token, N); // [N, n_token, num_kv_heads, head_dim] @@ -1658,7 +1796,7 @@ namespace LLM { 1.f, 32.f, 1.f); - } else if (arch == LLMArch::LLADA2_MOE) { + } else if (arch == LLMArch::LLADA2_MOE || arch == LLMArch::QWEN3_5) { // LLaDA2 slices the head (query[..., :rotary_dim]) instead of zero-padding // inv_freq like gemma does, so rotate_half pairs i with i + rotary_dim/2 and the // frequencies use rotary_dim as the exponent denominator. Passing n_dims = @@ -1734,6 +1872,9 @@ namespace LLM { x = ggml_ext_attention_ext(ctx, q, k, v, num_heads, attention_mask, true, ctx->flash_attn_enabled); // [N, n_token, hidden_size] } + if (output_gate) { + x = ggml_mul(ctx->ggml_ctx, x, ggml_sigmoid(ctx->ggml_ctx, output_gate)); + } x = out_proj->forward(ctx, x); // [N, n_token, hidden_size] return x; } @@ -1783,7 +1924,11 @@ namespace LLM { if (!config.sliding_attention.empty()) { sliding_attention = config.sliding_attention[layer_index % config.sliding_attention.size()]; } - blocks["self_attn"] = std::make_shared(config, sliding_attention == 0); + if (config.arch == LLMArch::QWEN3_5 && config.linear_attention_layers.count(layer_index)) { + blocks["linear_attn"] = std::make_shared(config); + } else { + blocks["self_attn"] = std::make_shared(config, sliding_attention == 0); + } if (config.arch == LLMArch::GPT_OSS_20B) { blocks["mlp"] = std::make_shared(config); } else if ((config.arch == LLMArch::LLADA2_MOE || config.arch == LLMArch::BAILING_MOE) && layer_index >= config.first_k_dense_replace) { @@ -1813,7 +1958,7 @@ namespace LLM { ggml_tensor* rope_cos = nullptr, ggml_tensor* rope_sin = nullptr) { // x: [N, n_token, hidden_size] - auto self_attn = std::dynamic_pointer_cast(blocks["self_attn"]); + auto self_attn = blocks.count("self_attn") ? std::dynamic_pointer_cast(blocks["self_attn"]) : nullptr; auto input_layernorm = std::dynamic_pointer_cast(blocks["input_layernorm"]); auto pre_ffw_norm = std::dynamic_pointer_cast(blocks[pre_ffw_norm_name]); std::shared_ptr post_attention_norm = nullptr; @@ -1833,7 +1978,11 @@ namespace LLM { auto residual = x; x = input_layernorm->forward(ctx, x); - x = self_attn->forward(ctx, x, input_pos, block_attention_mask, rope_index, rope_cos, rope_sin); + if (blocks.count("linear_attn")) { + x = std::dynamic_pointer_cast(blocks["linear_attn"])->forward(ctx, x); + } else { + x = self_attn->forward(ctx, x, input_pos, block_attention_mask, rope_index, rope_cos, rope_sin); + } if (post_attention_norm != nullptr) { x = post_attention_norm->forward(ctx, x); } @@ -2360,6 +2509,7 @@ namespace LLM { if (config.arch == LLMArch::MISTRAL_SMALL_3_2 || config.arch == LLMArch::MINISTRAL_3_3B || config.arch == LLMArch::QWEN3 || + config.arch == LLMArch::QWEN3_5 || config.arch == LLMArch::GEMMA3_12B || config.arch == LLMArch::GEMMA4_12B || config.arch == LLMArch::GEMMA2_2B || diff --git a/src/model/vae/minimax_h3_audio_vae.hpp b/src/model/vae/minimax_h3_audio_vae.hpp index 300826a03..c15055947 100644 --- a/src/model/vae/minimax_h3_audio_vae.hpp +++ b/src/model/vae/minimax_h3_audio_vae.hpp @@ -373,33 +373,22 @@ namespace MiniMaxH3 { auto pre = std::dynamic_pointer_cast(blocks["pre_block"]); auto mean_proj = std::dynamic_pointer_cast(blocks["mean_proj"]); - // GGML's batched conv1d storage interleaves the stream dimension - // with output channels. Subsequent layers then read stereo samples - // as adjacent feature channels. Run each mono stream independently, - // matching PyTorch's reshape(B*S, 1, samples), and concatenate only - // the completed normalized latents. const int64_t streams = waveform->ne[2] * waveform->ne[3]; waveform = ggml_reshape_3d(ctx->ggml_ctx, waveform, waveform->ne[0], 1, streams); - ggml_tensor* stereo_z = nullptr; - for (int64_t stream = 0; stream < streams; ++stream) { - auto mono = ggml_ext_slice(ctx->ggml_ctx, waveform, 2, stream, stream + 1); - auto x = encoder->forward(ctx, mono); - x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); - x = pre->forward(ctx, x); - x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); - auto z = mean_proj->forward(ctx, x); - - auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1); - auto std = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1); - z = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std); - z = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 2, 1, 3)); - stereo_z = stereo_z == nullptr ? z : ggml_concat(ctx->ggml_ctx, stereo_z, z, 1); - } - return stereo_z; + auto x = encoder->forward(ctx, waveform); + x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); + x = pre->forward(ctx, x); + x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); + auto z = mean_proj->forward(ctx, x); + + auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1); + auto std = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1); + z = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std); + return ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 2, 1, 3)); } ggml_tensor* decode(GGMLRunnerContext* ctx, ggml_tensor* latent) { diff --git a/src/model_io/gguf_io.cpp b/src/model_io/gguf_io.cpp index 3e5744878..8120131ea 100644 --- a/src/model_io/gguf_io.cpp +++ b/src/model_io/gguf_io.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -19,7 +20,7 @@ static void set_error(std::string* error, const std::string& message) { } bool is_gguf_file(const std::string& file_path) { - std::ifstream file(file_path, std::ios::binary); + std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary); if (!file.is_open()) { return false; } @@ -178,7 +179,7 @@ bool GGUFStreamingWriter::write_metadata(const std::string& file_path, } LOG_INFO("trying to save tensors to %s", file_path.c_str()); - FILE* file = fopen(file_path.c_str(), "wb+"); + FILE* file = ggml_fopen(file_path.c_str(), "wb+"); if (file == nullptr) { set_error(error, "failed to open output file '" + file_path + "'"); close(); diff --git a/src/model_io/gguf_reader_ext.h b/src/model_io/gguf_reader_ext.h index b55d49f11..f02fe3c6f 100644 --- a/src/model_io/gguf_reader_ext.h +++ b/src/model_io/gguf_reader_ext.h @@ -2,6 +2,7 @@ #define __SD_MODEL_IO_GGUF_READER_EXT_H__ #include +#include #include #include #include @@ -184,7 +185,7 @@ class GGUFReader { public: bool load(const std::string& file_path) { - std::ifstream fin(file_path, std::ios::binary | std::ios::ate); + std::ifstream fin(std::filesystem::u8path(file_path), std::ios::binary | std::ios::ate); if (!fin) { LOG_ERROR("failed to open '%s'", file_path.c_str()); return false; diff --git a/src/model_io/safetensors_io.cpp b/src/model_io/safetensors_io.cpp index 3e6974237..cf424af92 100644 --- a/src/model_io/safetensors_io.cpp +++ b/src/model_io/safetensors_io.cpp @@ -26,15 +26,15 @@ static void set_error(std::string* error, const std::string& message) { } static std::string resolve_index_shard_path(const std::string& index_path, const std::string& shard_path) { - fs::path shard_fs_path(shard_path); + fs::path shard_fs_path = fs::u8path(shard_path); if (shard_fs_path.is_absolute()) { - return shard_fs_path.lexically_normal().string(); + return shard_fs_path.lexically_normal().u8string(); } - return (fs::path(index_path).parent_path() / shard_fs_path).lexically_normal().string(); + return (fs::u8path(index_path).parent_path() / shard_fs_path).lexically_normal().u8string(); } bool is_safetensors_file(const std::string& file_path) { - std::ifstream file(file_path, std::ios::binary); + std::ifstream file(fs::u8path(file_path), std::ios::binary); if (!file.is_open()) { return false; } @@ -153,7 +153,7 @@ bool read_safetensors_file(const std::string& file_path, std::string* error, std::map* metadata, std::set* tensor_names) { - std::ifstream file(file_path, std::ios::binary); + std::ifstream file(fs::u8path(file_path), std::ios::binary); if (!file.is_open()) { set_error(error, "failed to open '" + file_path + "'"); return false; @@ -418,7 +418,7 @@ bool read_safetensors_index_file(const std::string& file_path, std::string* error) { shard_paths.clear(); - std::ifstream file(file_path); + std::ifstream file(fs::u8path(file_path)); if (!file.is_open()) { set_error(error, "failed to open '" + file_path + "'"); return false; @@ -522,7 +522,7 @@ bool write_safetensors_file(const std::string& file_path, const std::string header_str = header.dump(); - std::ofstream file(file_path, std::ios::binary); + std::ofstream file(fs::u8path(file_path), std::ios::binary); if (!file.is_open()) { set_error(error, "failed to open '" + file_path + "' for writing"); return false; @@ -596,7 +596,7 @@ bool SafetensorsStreamingWriter::write_metadata(const std::string& file_path, data_start_ = ST_HEADER_SIZE_LEN + header_str.size(); LOG_INFO("trying to save tensors to %s", file_path.c_str()); - std::ofstream file(file_path, std::ios::binary | std::ios::trunc); + std::ofstream file(fs::u8path(file_path), std::ios::binary | std::ios::trunc); if (!file.is_open()) { set_error(error, "failed to open '" + file_path + "' for writing"); return false; diff --git a/src/model_io/torch_legacy_io.cpp b/src/model_io/torch_legacy_io.cpp index 3170fe8d4..163e6649e 100644 --- a/src/model_io/torch_legacy_io.cpp +++ b/src/model_io/torch_legacy_io.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -98,7 +99,7 @@ static std::string torch_legacy_diagnostics(const std::string& file_path, const bool read_torch_legacy_file(const std::string& file_path, std::vector& tensor_storages, std::string* error) { - std::ifstream file(file_path, std::ios::binary); + std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary); if (!file.is_open()) { set_error(error, "failed to open '" + file_path + "'"); return false; diff --git a/src/model_loader.cpp b/src/model_loader.cpp index 4dc2dcbca..90ab65eed 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -333,7 +334,7 @@ bool ModelLoader::init_from_safetensors_index_file(const std::string& file_path, } } - std::ifstream index_file(file_path); + std::ifstream index_file(std::filesystem::u8path(file_path)); const auto index = nlohmann::json::parse(index_file); for (const auto& entry : index.at("weight_map").items()) { const auto expected = (std::filesystem::u8path(file_path).parent_path() / @@ -443,6 +444,11 @@ SDVersion ModelLoader::get_sd_version() const { return VERSION_FLUX3; } + if (tensor_storage_map.count("model.diffusion_model.video_embedder.weight") && + tensor_storage_map.count("model.diffusion_model.shared_block_modulation.weight")) { + return VERSION_MOSS_IMAGE; + } + bool has_multiple_encoders = false; bool is_unet = false; @@ -1136,7 +1142,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, return; } } else if (!mmapped) { - file.open(file_path, std::ios::binary); + file.open(std::filesystem::u8path(file_path), std::ios::binary); if (!file.is_open()) { LOG_ERROR("failed to open '%s'", file_path.c_str()); failed = true; diff --git a/src/model_loader.h b/src/model_loader.h index 63ecfb63b..aae359099 100644 --- a/src/model_loader.h +++ b/src/model_loader.h @@ -105,6 +105,8 @@ class ModelLoader { bool del_file(FileId id); uint64_t file_revision(FileId id) const; std::string file_path(FileId id) const; + std::map file_metadata(FileId id) const; + String2TensorStorage file_tensors(FileId id) const; String2TensorStorage file_tensors(FileId id, SDVersion version) const; bool load_file_tensors(FileId id, SDVersion version, on_new_tensor_cb_t callback, const std::set& names, bool use_mmap = false) const; bool refresh_files(bool include_isolated = true); diff --git a/src/model_loader_files.cpp b/src/model_loader_files.cpp index 8e838b119..9ac5456cc 100644 --- a/src/model_loader_files.cpp +++ b/src/model_loader_files.cpp @@ -324,6 +324,24 @@ ModelLoader ModelLoader::file_reader(FileId id, SDVersion version) const { return reader; } +std::map ModelLoader::file_metadata(FileId id) const { + for (const auto& file : files_) { + if (file.id == id) { + return file.metadata; + } + } + return {}; +} + +String2TensorStorage ModelLoader::file_tensors(FileId id) const { + for (const auto& file : files_) { + if (file.id == id) { + return file.tensors; + } + } + return {}; +} + String2TensorStorage ModelLoader::file_tensors(FileId id, SDVersion version) const { return file_reader(id, version).tensor_storage_map; } diff --git a/src/model_manager.cpp b/src/model_manager.cpp index a19614314..05ee55f67 100644 --- a/src/model_manager.cpp +++ b/src/model_manager.cpp @@ -1060,32 +1060,65 @@ bool ModelManager::load_tensor_parts(TensorState& state) { bool ModelManager::load_tensors(const std::vector& states) { using ReadGroup = std::pair; - using ReadBatch = std::map>; + using ReadBatch = std::map>; std::map> groups; - for (auto* state : states) { - if (state == nullptr) - continue; - if (state->sources.size() > 1) { - if (!load_tensor_parts(*state)) { - return false; - } - continue; + size_t part_count = 0; + for (const auto* state : states) { + if (state != nullptr && state->sources.size() > 1 && ggml_backend_buffer_is_host(state->tensor->buffer)) { + part_count += state->sources.size(); } - auto& batches = groups[{state->source_file, state->source_version}]; - // Others copy from the first destination: a type conflict or a repacked first destination needs another batch. + } + auto parts_ctx = std::unique_ptr( + part_count > 0 ? ggml_init({part_count * ggml_tensor_overhead(), nullptr, true}) : nullptr, ggml_free); + if (part_count > 0 && !parts_ctx) { + return false; + } + auto add_destination = [&](const TensorState& state, const std::string& name, ggml_tensor* tensor) { + auto& batches = groups[{state.source_file, state.source_version}]; + // The loader supplies one destination per name; incompatible destinations need another batch. auto batch = std::find_if(batches.begin(), batches.end(), [&](const ReadBatch& candidate) { - auto found = candidate.find(state->name); + auto found = candidate.find(name); if (found == candidate.end()) { return true; } - const ggml_tensor* first = found->second.front()->tensor; - return first->type == state->tensor->type && !sd_weight_may_be_cpu_repacked(first); + // Others copy from the first destination, so a repacked first destination needs another batch. + const ggml_tensor* first = found->second.front(); + return first->type == tensor->type && ggml_are_same_shape(first, tensor) && + !sd_weight_may_be_cpu_repacked(first); }); if (batch == batches.end()) { batches.emplace_back(); batch = std::prev(batches.end()); } - (*batch)[state->name].push_back(state); + (*batch)[name].push_back(tensor); + }; + for (auto* state : states) { + if (state == nullptr) + continue; + if (state->sources.size() > 1) { + if (ggml_backend_buffer_is_host(state->tensor->buffer)) { + const size_t size = ggml_nbytes(state->tensor); + size_t offset = 0; + for (const auto& source : state->sources) { + auto* part = ggml_new_tensor(parts_ctx.get(), state->tensor->type, source.n_dims, source.ne); + const size_t part_size = ggml_nbytes(part); + if (part_size > size - offset) { + return false; + } + part->data = static_cast(state->tensor->data) + offset; + part->buffer = state->tensor->buffer; + add_destination(*state, source.name, part); + offset += part_size; + } + if (offset != size) { + return false; + } + } else if (!load_tensor_parts(*state)) { + return false; + } + continue; + } + add_destination(*state, state->name, state->tensor); } for (auto& group : groups) { for (auto& batch : group.second) { @@ -1099,7 +1132,7 @@ bool ModelManager::load_tensors(const std::vector& states) { auto found = batch.find(source.name); if (found == batch.end()) return true; - *dst = found->second.front()->tensor; + *dst = found->second.front(); std::lock_guard lock(mutex); loaded.insert(source.name); return true; @@ -1110,15 +1143,18 @@ bool ModelManager::load_tensors(const std::vector& states) { if (!success || loaded != names) return false; for (auto& entry : batch) { - auto* first = entry.second.front()->tensor; - for (auto* state : entry.second) { - if (state->tensor != first) - ggml_backend_tensor_copy(first, state->tensor); - state->loaded_to_params_backend = true; + auto* first = entry.second.front(); + for (auto* tensor : entry.second) { + if (tensor != first) + ggml_backend_tensor_copy(first, tensor); } } } } + for (auto* state : states) { + if (state != nullptr) + state->loaded_to_params_backend = true; + } return true; } diff --git a/src/name_conversion.cpp b/src/name_conversion.cpp index 8b286e813..6a519ba21 100644 --- a/src/name_conversion.cpp +++ b/src/name_conversion.cpp @@ -1013,6 +1013,16 @@ static std::string convert_diffusers_dit_to_original_pixart(std::string name) { return prefix + name; } +static std::string convert_moss_image_dit_name(std::string name) { + static const std::vector> name_map = { + {".proj.q.weight", ".proj.qkv.weight"}, + {".proj.k.weight", ".proj.qkv.weight.1"}, + {".proj.v.weight", ".proj.qkv.weight.2"}, + }; + replace_with_name_map(name, name_map); + return name; +} + static std::string convert_ming_image_dit_name(std::string name) { static const std::vector> name_map = { {"all_x_embedder.2-1.", "x_embedder."}, @@ -1024,6 +1034,24 @@ static std::string convert_ming_image_dit_name(std::string name) { return name; } +std::string convert_diffusers_dit_to_original_minimax_h3(std::string name) { + static const std::vector> prefix_map = { + {"transformer_blocks.", "blocks."}, + {"token_refiner.refiner_blocks.", "token_refiner.blocks."}, + }; + static const std::vector> name_map = { + {".attn.to_q.weight", ".attn.qkv_proj.weight"}, + {".attn.to_k.weight", ".attn.qkv_proj.weight.1"}, + {".attn.to_v.weight", ".attn.qkv_proj.weight.2"}, + {".attn.to_out.0.", ".attn.out_proj."}, + {".ff.net.0.proj.", ".mlp.fc1."}, + {".ff.net.2.", ".mlp.fc2."}, + }; + replace_with_prefix_map(name, prefix_map); + replace_with_name_map(name, name_map); + return name; +} + std::string convert_diffusion_model_name(std::string name, std::string prefix, SDVersion version) { if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) { name = convert_diffusers_unet_to_original_sd1(name); @@ -1035,6 +1063,8 @@ std::string convert_diffusion_model_name(std::string name, std::string prefix, S name = convert_diffusers_dit_to_original_flux(name); } else if (sd_version_is_hunyuan_video(version)) { name = convert_hunyuan_video_to_original_flux(name); + } else if (version == VERSION_MOSS_IMAGE) { + name = convert_moss_image_dit_name(name); } else if (version == VERSION_MING_IMAGE) { name = convert_ming_image_dit_name(name); } else if (sd_version_is_z_image(version) || sd_version_is_z_image_l2p(version)) { @@ -1047,6 +1077,8 @@ std::string convert_diffusion_model_name(std::string name, std::string prefix, S name = convert_diffusers_dit_to_original_krea2(name); } else if (sd_version_is_pixart(version)) { name = convert_diffusers_dit_to_original_pixart(name); + } else if (sd_version_is_minimax_h3(version)) { + name = convert_diffusers_dit_to_original_minimax_h3(name); } return name; } @@ -1663,6 +1695,26 @@ std::string convert_tensor_name(std::string name, SDVersion version) { replace_with_prefix_map(name, prefix_map); + if (version == VERSION_MOSS_IMAGE) { + const std::string te_prefix = "text_encoders.llm."; + if (starts_with(name, te_prefix + "blk.")) { + static const std::vector> qwen35_name_map = { + {"attn_post_norm.", "post_attention_layernorm."}, + {"post_attention_norm.", "post_attention_layernorm."}, + {"attn_qkv.", "linear_attn.in_proj_qkv."}, + {"attn_gate.", "linear_attn.in_proj_z."}, + {"ssm_conv1d.", "linear_attn.conv1d."}, + {"ssm_dt.bias", "linear_attn.dt_bias"}, + {"ssm_norm.", "linear_attn.norm."}, + {"ssm_out.", "linear_attn.out_proj."}, + {"ssm_alpha.", "linear_attn.in_proj_a."}, + {"ssm_beta.", "linear_attn.in_proj_b."}, + {"ssm_a", "linear_attn.A"}, + }; + replace_with_name_map(name, qwen35_name_map); + } + } + if (version == VERSION_QWEN_IMAGE_2_1 || sd_version_is_boogu_image(version) || sd_version_is_krea2(version) || sd_version_is_mage_flow(version) || sd_version_is_minimax_h3(version)) { const std::string hf_vision_prefix = "text_encoders.llm.model.visual."; if (starts_with(name, hf_vision_prefix)) { diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index 62286b0a7..1297ca020 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -113,6 +113,7 @@ const char* model_version_to_str[] = { "Z-Image L2P", "Iris-3B", "Flux.3 Image Klein", + "MOSS-Image", }; static_assert(VERSION_COUNT == sizeof(model_version_to_str) / sizeof(model_version_to_str[0]), @@ -1058,9 +1059,15 @@ void StableDiffusionGGML::configure_weight_loading() { const auto& model_loader = model_manager->loader(); const auto wtype_stat = model_loader.get_wtype_stat(); bool have_int8_tensorwise = false; + bool have_quant_type_rule = false; for (const auto& [_, tensor_storage] : model_loader.get_tensor_storage_map()) { if (tensor_storage.is_int8_tensorwise) { have_int8_tensorwise = true; + } + if (tensor_storage.expected_type < GGML_TYPE_COUNT && ggml_is_quantized(tensor_storage.expected_type)) { + have_quant_type_rule = true; + } + if (have_int8_tensorwise && have_quant_type_rule) { break; } } @@ -1077,18 +1084,8 @@ void StableDiffusionGGML::configure_weight_loading() { const bool params_offloaded = params_backend_for(SDBackendModule::DIFFUSION) != backend_for(SDBackendModule::DIFFUSION); const bool streaming_constrained = params_offloaded || backend_manager.params_backend_is_disk(SDBackendModule::DIFFUSION); - // On CPU, weights quantized at load (--type / --tensor-type-rules) cannot take a merged LoRA. - const ggml_type target_wtype = sd_type_to_ggml_type(sd_ctx_params->wtype); - bool quantized_on_load = target_wtype < GGML_TYPE_COUNT && ggml_is_quantized(target_wtype); - for (const auto& rule : parse_tensor_type_rules(SAFE_STR(sd_ctx_params->tensor_type_rules))) { - quantized_on_load = quantized_on_load || ggml_is_quantized(rule.second); - } - bool lora_module_on_cpu = false; - for (SDBackendModule module : {SDBackendModule::DIFFUSION, SDBackendModule::TE, SDBackendModule::VAE}) { - lora_module_on_cpu = lora_module_on_cpu || sd_backend_is_cpu(backend_for(module)); - } - const bool cpu_quantized_on_load = quantized_on_load && lora_module_on_cpu; - if (have_quantized_weight || cpu_quantized_on_load || streaming_constrained || row_split_active()) { + + if (have_quant_type_rule || have_quantized_weight || streaming_constrained || row_split_active()) { apply_lora_immediately = false; } else { apply_lora_immediately = true; @@ -1522,9 +1519,10 @@ bool StableDiffusionGGML::build_denoiser() { sd_version_is_boogu_image(version) || sd_version_is_pid(version) || sd_version_is_ideogram4(version) || - version == VERSION_IRIS) { + version == VERSION_IRIS || + version == VERSION_MOSS_IMAGE) { pred_type = FLOW_PRED; - if (sd_version_is_wan(version)) { + if (sd_version_is_wan(version) || version == VERSION_MOSS_IMAGE) { default_flow_shift = 5.f; } else if (sd_version_is_hunyuan_video(version)) { default_flow_shift = 7.f; @@ -2676,7 +2674,7 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr StableDiffusionGGML::sample(const std::shared_ptr* uncond_skip_layers = nullptr; + const std::vector moss_skip_layers = {22}; if (is_skiplayer_step && slg_uncond) { LOG_VERBOSE("Skipping layers at uncond step %d\n", step); uncond_skip_layers = &skip_layer_guidance.layers(); + } else if (version == VERSION_MOSS_IMAGE) { + uncond_skip_layers = &moss_skip_layers; } uncond_out = run_condition(uncond, uncond.c_concat.empty() ? nullptr : &uncond.c_concat, diff --git a/src/pipeline/model_builders.cpp b/src/pipeline/model_builders.cpp index 494567904..4b435170f 100644 --- a/src/pipeline/model_builders.cpp +++ b/src/pipeline/model_builders.cpp @@ -31,6 +31,7 @@ #include "model/diffusion/minit2i.hpp" #include "model/diffusion/mmdit.hpp" #include "model/diffusion/model.hpp" +#include "model/diffusion/moss_image.hpp" #include "model/diffusion/pid.hpp" #include "model/diffusion/pixart.hpp" #include "model/diffusion/qwen_image.hpp" @@ -468,6 +469,11 @@ namespace sd::model_builders { tensor_storage_map, version, "", false, weight_manager, tokenizers); result.diffusion = std::make_shared(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION), tensor_storage_map, "model.diffusion_model", weight_manager); + } else if (version == VERSION_MOSS_IMAGE) { + result.conditioner = std::make_shared(ctx.backends.runtime_backend(SDBackendModule::TE), + tensor_storage_map, version, "", false, weight_manager, tokenizers); + result.diffusion = std::make_shared(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION), + tensor_storage_map, "model.diffusion_model", weight_manager); } else { // SD1.x SD2.x SDXL std::map embbeding_map; for (uint32_t i = 0; i < sd_ctx_params->embedding_count; i++) { diff --git a/src/runtime/imatrix.cpp b/src/runtime/imatrix.cpp index 3746a3711..185832cb2 100644 --- a/src/runtime/imatrix.cpp +++ b/src/runtime/imatrix.cpp @@ -10,6 +10,7 @@ #include #include #include +#include static IMatrixCollector imatrix_collector; @@ -280,7 +281,7 @@ void IMatrixCollector::save_imatrix(std::string fname, int ncall) const { LOG_WARN("storing only %zu out of %zu entries\n", to_store.size(), stats_.size()); } - std::ofstream out(fname, std::ios::binary); + std::ofstream out(std::filesystem::u8path(fname), std::ios::binary); out.write((const char*)&n_entries, sizeof(n_entries)); for (const auto& name : to_store) { const auto& stat = stats_.at(name); @@ -304,7 +305,7 @@ void IMatrixCollector::save_imatrix(std::string fname, int ncall) const { } bool IMatrixCollector::load_imatrix(const char* fname) { - std::ifstream in(fname, std::ios::binary); + std::ifstream in(std::filesystem::u8path(fname), std::ios::binary); if (!in) { LOG_ERROR("failed to open %s\n", fname); return false; diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 1cc4c09d3..d476f92d6 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -704,6 +704,13 @@ SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx) { return sd_version_supports_video_generation(sd_ctx->sd->version); } +SD_API bool sd_ctx_supports_ref_image_conditioning(const sd_ctx_t* sd_ctx) { + if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { + return false; + } + return sd_version_supports_ref_image_conditioning(sd_ctx->sd->version); +} + SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path) { if (sd_ctx == nullptr || sd_ctx->sd == nullptr || path == nullptr) { return false; diff --git a/src/tokenizers/hf_tokenizer.cpp b/src/tokenizers/hf_tokenizer.cpp index f8daf6d2a..6e6401819 100644 --- a/src/tokenizers/hf_tokenizer.cpp +++ b/src/tokenizers/hf_tokenizer.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -495,7 +496,7 @@ struct HFTokenizer::Impl { HFTokenizer::HFTokenizer(const std::string& path) : impl_(new Impl) { - std::ifstream stream(path, std::ios::binary); + std::ifstream stream(std::filesystem::u8path(path), std::ios::binary); tokenizer_require(stream.good(), "cannot open " + path); TokenizerJSON config; stream >> config; diff --git a/start-server.bat b/start-server.bat new file mode 100644 index 000000000..60ea9e180 --- /dev/null +++ b/start-server.bat @@ -0,0 +1,45 @@ +@echo off +setlocal DisableDelayedExpansion + +set "SD_SERVER_PATH=%SD_SERVER_PATH%" +set "MODEL_DIR=./models" +set "COMMANDLINE_ARGS=" + +if exist "%~dp0start-server.local.bat" ( + call "%~dp0start-server.local.bat" + if errorlevel 1 exit /b 1 +) + +if defined SD_SERVER_PATH goto explicit + +for %%F in ( + "%~dp0sd-server.exe" + "%~dp0bin\Release\sd-server.exe" + "%~dp0bin\sd-server.exe" + "%~dp0build\bin\Release\sd-server.exe" + "%~dp0build\bin\sd-server.exe" +) do ( + if exist "%%~F" ( + set "SD_SERVER_PATH=%%~F" + goto run + ) +) + +>&2 echo sd-server.exe was not found. Build it first or set SD_SERVER_PATH to its executable path. +exit /b 1 + +:explicit +if exist "%SD_SERVER_PATH%" goto run +>&2 echo SD_SERVER_PATH does not exist: "%SD_SERVER_PATH%" +exit /b 1 + +:run +if "%~1"=="--version" goto version +echo Running: "%SD_SERVER_PATH%" --model-dir "%MODEL_DIR%" %COMMANDLINE_ARGS% %* +"%SD_SERVER_PATH%" --model-dir "%MODEL_DIR%" %COMMANDLINE_ARGS% %* +exit /b %errorlevel% + +:version +echo Running: "%SD_SERVER_PATH%" %* +"%SD_SERVER_PATH%" %* +exit /b %errorlevel% diff --git a/start-server.sh b/start-server.sh new file mode 100644 index 000000000..b73dc2745 --- /dev/null +++ b/start-server.sh @@ -0,0 +1,53 @@ +#!/bin/sh +set -eu + +SD_SERVER_PATH="${SD_SERVER_PATH:-}" +MODEL_DIR="./models" +COMMANDLINE_ARGS="" + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +if [ -f "$script_dir/start-server.local.sh" ]; then + . "$script_dir/start-server.local.sh" +fi + +if [ "${1:-}" != "--version" ]; then + # Split configured arguments on whitespace without expanding wildcards. + set -f + set -- --model-dir "$MODEL_DIR" $COMMANDLINE_ARGS "$@" +fi + +run_server() { + printf 'Running:' + for arg do + printf " '%s'" "$(printf '%s' "$arg" | sed "s/'/'\\\\''/g")" + done + printf '\n' + exec "$@" +} + +if [ -n "$SD_SERVER_PATH" ]; then + case "$SD_SERVER_PATH" in + */*) ;; + *) SD_SERVER_PATH="./$SD_SERVER_PATH" ;; + esac + if [ ! -f "$SD_SERVER_PATH" ] || [ ! -x "$SD_SERVER_PATH" ]; then + printf 'SD_SERVER_PATH is not an executable file: %s\n' "$SD_SERVER_PATH" >&2 + exit 1 + fi + run_server "$SD_SERVER_PATH" "$@" +fi + +for server in \ + "$script_dir/sd-server" \ + "$script_dir/bin/Release/sd-server" \ + "$script_dir/bin/sd-server" \ + "$script_dir/build/bin/Release/sd-server" \ + "$script_dir/build/bin/sd-server" +do + if [ -f "$server" ] && [ -x "$server" ]; then + run_server "$server" "$@" + fi +done + +printf '%s\n' 'sd-server was not found. Build it first or set SD_SERVER_PATH to its executable path.' >&2 +exit 1