From b6f53b718f996e021822dfbf89c6551ce20dbcbd Mon Sep 17 00:00:00 2001 From: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:28:34 +0100 Subject: [PATCH 1/9] Adding GPU Test Runner Signed-off-by: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> --- .github/workflows/cicd_tests_gpu.yml | 83 ++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 .github/workflows/cicd_tests_gpu.yml diff --git a/.github/workflows/cicd_tests_gpu.yml b/.github/workflows/cicd_tests_gpu.yml new file mode 100644 index 0000000000..e4b8c3e224 --- /dev/null +++ b/.github/workflows/cicd_tests_gpu.yml @@ -0,0 +1,83 @@ +name: tests + +on: + # quick tests for pull requests and the releasing branches + push: + branches: + - dev + - main + - releasing/* + pull_request: + branches: + - dev + - main + - releasing/* + +permissions: + contents: read + +concurrency: + # automatically cancel the previously triggered workflows when there's a newer version + group: cicd-tests-gpu-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +# Supported versions of Python and PyTorch are listed here for use below and as documentation +env: + # supported versions of Python + PYTHON_VER1: '3.10' + PYTHON_VER2: '3.11' + PYTHON_VER3: '3.12' + PYTHON_VER4: '3.13' + # PYTHON_VER5: '3.14' # TODO: not compatible with Torchscript, re-enable once Torchscript removed from MONAI + # supported versions of PyTorch + PYTORCH_VER1: '2.8.0' + PYTORCH_VER2: '2.9.1' + PYTORCH_VER3: '2.10.0' + PYTORCH_VER4: '2.11.0' + TORCHVISION_VER1: '0.23.0' # used for testing with lowest PyTorch version (PYTORCH_VER1), update as needed + + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python # https://github.com/Project-MONAI/MONAI/issues/4354 + +# These jobs run the CICD tests, type checking, and testing packaging. These use the minimum supported versions of +# Python and PyTorch in many places using the above environment variables but also hard coded where necessary. +# When support is dropped for a version it is important to update these as appropriate. + +jobs: + + full-dep: # Test with full dependencies installed for different OS runners + runs-on: [self-hosted, linux, gpu] #ubuntu-24.04-gpu] + strategy: + fail-fast: false + timeout-minutes: 120 + steps: + - uses: actions/checkout@v7 + - name: Set up Python ${{ env.PYTHON_VER1 }} + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VER1 }} + cache: 'pip' + - name: Prepare pip wheel + run: | + which python + python -m pip install --upgrade pip wheel + - name: Install torch gpu + run: | + # install the lowest version of PyTorch supported + python -m pip install torch==${PYTORCH_VER1} torchvision==${TORCHVISION_VER1} + shell: bash + - name: Install the complete dependencies + run: | + python -m pip install --pre -U itk + python -m pip install --user --upgrade pip wheel tomli + python monai/config/print_dependencies.py build-system | xargs pip install --no-build-isolation + python -m pip install --no-build-isolation .[all,testing] + python -m pip list + shell: bash + - name: Run compiled tests + run: | + python -m pip uninstall -y monai + BUILD_MONAI=1 python -m pip install --no-build-isolation -e . # compile the cpp extensions + python -c 'import monai._C' > /dev/null + nvidia-smi || true + python monai/config/check_env.py --env --MONAI + python -m unittest -v From c9fdb8366ecc9efef3f1416f7219dd36fce37050 Mon Sep 17 00:00:00 2001 From: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:38:06 +0100 Subject: [PATCH 2/9] Fix Signed-off-by: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> --- .github/workflows/cicd_tests_gpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd_tests_gpu.yml b/.github/workflows/cicd_tests_gpu.yml index e4b8c3e224..e3df5cc217 100644 --- a/.github/workflows/cicd_tests_gpu.yml +++ b/.github/workflows/cicd_tests_gpu.yml @@ -79,5 +79,5 @@ jobs: BUILD_MONAI=1 python -m pip install --no-build-isolation -e . # compile the cpp extensions python -c 'import monai._C' > /dev/null nvidia-smi || true - python monai/config/check_env.py --env --MONAI + python monai/config/check_env.py --env --monai python -m unittest -v From f0d79b9744a5c0651e4fb1a645a5983402ecf48e Mon Sep 17 00:00:00 2001 From: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:58:47 +0100 Subject: [PATCH 3/9] Update Signed-off-by: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> --- .github/workflows/cicd_tests_gpu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd_tests_gpu.yml b/.github/workflows/cicd_tests_gpu.yml index e3df5cc217..23927c51ab 100644 --- a/.github/workflows/cicd_tests_gpu.yml +++ b/.github/workflows/cicd_tests_gpu.yml @@ -44,11 +44,11 @@ env: jobs: - full-dep: # Test with full dependencies installed for different OS runners + full-dep-gpu: # Test with full dependencies installed for different OS runners runs-on: [self-hosted, linux, gpu] #ubuntu-24.04-gpu] strategy: fail-fast: false - timeout-minutes: 120 + timeout-minutes: 360 steps: - uses: actions/checkout@v7 - name: Set up Python ${{ env.PYTHON_VER1 }} From 9c598ffbcd25d32ce9ac7870f15819f37f93d84c Mon Sep 17 00:00:00 2001 From: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:58:58 +0100 Subject: [PATCH 4/9] Fix retinanet for now Signed-off-by: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> --- .../detection/networks/retinanet_detector.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/monai/apps/detection/networks/retinanet_detector.py b/monai/apps/detection/networks/retinanet_detector.py index 9b9bf26911..1b73108756 100644 --- a/monai/apps/detection/networks/retinanet_detector.py +++ b/monai/apps/detection/networks/retinanet_detector.py @@ -41,7 +41,7 @@ import warnings from collections.abc import Callable, Sequence -from typing import Any +from typing import TYPE_CHECKING, Any import torch from torch import Tensor, nn @@ -59,10 +59,13 @@ from monai.networks.nets import resnet from monai.utils import BlendMode, PytorchPadMode, ensure_tuple_rep, optional_import -BalancedPositiveNegativeSampler, _ = optional_import( - "torchvision.models.detection._utils", name="BalancedPositiveNegativeSampler" -) -Matcher, _ = optional_import("torchvision.models.detection._utils", name="Matcher") +if TYPE_CHECKING: + from torchvision.models.detection._utils import BalancedPositiveNegativeSampler, Matcher +else: + BalancedPositiveNegativeSampler, _ = optional_import( + "torchvision.models.detection._utils", name="BalancedPositiveNegativeSampler" + ) + Matcher, _ = optional_import("torchvision.models.detection._utils", name="Matcher") class RetinaNetDetector(nn.Module): @@ -769,10 +772,11 @@ def compute_anchor_matched_idxs( # BELOW_LOW_THRESHOLD = -1, BETWEEN_THRESHOLDS = -2 if isinstance(self.proposal_matcher, Matcher): # if torchvision matcher + matcher: Matcher = self.proposal_matcher match_quality_matrix = self.box_overlap_metric( targets_per_image[self.target_box_key].to(anchors_per_image.device), anchors_per_image ) - matched_idxs_per_image = self.proposal_matcher(match_quality_matrix) + matched_idxs_per_image = matcher(match_quality_matrix) elif isinstance(self.proposal_matcher, ATSSMatcher): # if monai ATSS matcher match_quality_matrix, matched_idxs_per_image = self.proposal_matcher( From 94fc7981eeb72b32e924564b3e8db3c8540d73b2 Mon Sep 17 00:00:00 2001 From: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> Date: Sun, 25 Oct 2026 10:49:02 +0000 Subject: [PATCH 5/9] Try quick tests first Signed-off-by: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> --- .github/workflows/cicd_tests.yml | 43 ++++++++++--------- .github/workflows/cicd_tests_gpu.yml | 34 +++++++++------ monai/engines/trainer.py | 6 +-- monai/engines/workflow.py | 2 +- .../test_integration_fast_train.py | 2 +- 5 files changed, 48 insertions(+), 39 deletions(-) diff --git a/.github/workflows/cicd_tests.yml b/.github/workflows/cicd_tests.yml index 28a420d8c8..354b246b11 100644 --- a/.github/workflows/cicd_tests.yml +++ b/.github/workflows/cicd_tests.yml @@ -70,29 +70,30 @@ jobs: run: | echo "skip=1" >> $GITHUB_OUTPUT - mkdir -p /tmp/check_changes - if ! git diff --name-only HEAD^..HEAD ':!docs' > /tmp/check_changes/changed_files - then - echo "skip=0" >> "$GITHUB_OUTPUT" - exit 0 - fi +# TEMPORARILY SKIP ALL CI TESTS, UNDO BEFORE MERGE + # mkdir -p /tmp/check_changes + # if ! git diff --name-only HEAD^..HEAD ':!docs' > /tmp/check_changes/changed_files + # then + # echo "skip=0" >> "$GITHUB_OUTPUT" + # exit 0 + # fi - while read i - do - echo "Checking for changes: $i" - if ! git show HEAD^:$i > /tmp/check_changes/$(basename $i) - then - echo "New file found, or 'git show' failed, running tests." - elif ! python monai/utils/compare_sources.py $i /tmp/check_changes/$(basename $i) - then - echo "Change found, running tests." - else - continue - fi + # while read i + # do + # echo "Checking for changes: $i" + # if ! git show HEAD^:$i > /tmp/check_changes/$(basename $i) + # then + # echo "New file found, or 'git show' failed, running tests." + # elif ! python monai/utils/compare_sources.py $i /tmp/check_changes/$(basename $i) + # then + # echo "Change found, running tests." + # else + # continue + # fi - echo "skip=0" >> $GITHUB_OUTPUT - break - done < /tmp/check_changes/changed_files + # echo "skip=0" >> $GITHUB_OUTPUT + # break + # done < /tmp/check_changes/changed_files static-checks: # Perform static type and other checks using runtests.sh runs-on: ubuntu-latest diff --git a/.github/workflows/cicd_tests_gpu.yml b/.github/workflows/cicd_tests_gpu.yml index 23927c51ab..04cdb9ccdd 100644 --- a/.github/workflows/cicd_tests_gpu.yml +++ b/.github/workflows/cicd_tests_gpu.yml @@ -24,20 +24,24 @@ concurrency: # Supported versions of Python and PyTorch are listed here for use below and as documentation env: # supported versions of Python - PYTHON_VER1: '3.10' - PYTHON_VER2: '3.11' - PYTHON_VER3: '3.12' - PYTHON_VER4: '3.13' + PYTHON_VER1: '3.11' + PYTHON_VER2: '3.12' + PYTHON_VER3: '3.13' + PYTHON_VER4: '3.14' # PYTHON_VER5: '3.14' # TODO: not compatible with Torchscript, re-enable once Torchscript removed from MONAI # supported versions of PyTorch - PYTORCH_VER1: '2.8.0' - PYTORCH_VER2: '2.9.1' - PYTORCH_VER3: '2.10.0' - PYTORCH_VER4: '2.11.0' - TORCHVISION_VER1: '0.23.0' # used for testing with lowest PyTorch version (PYTORCH_VER1), update as needed + PYTORCH_VER1: '2.12.1' + PYTORCH_VER2: '2.13.0' + PYTORCH_VER3: '2.14.0' + PYTORCH_VER4: '2.14.1' + TORCHVISION_VER1: '0.27.1' # used for testing with lowest PyTorch version (PYTORCH_VER1), update as needed + + TQDM_DISABLE: 1 # squelch TQDM output to reduce log file size PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python # https://github.com/Project-MONAI/MONAI/issues/4354 + BUILD_MONAI: 1 # always build MONAI and ensure it tries to use the compiled extensions + # These jobs run the CICD tests, type checking, and testing packaging. These use the minimum supported versions of # Python and PyTorch in many places using the above environment variables but also hard coded where necessary. # When support is dropped for a version it is important to update these as appropriate. @@ -73,11 +77,15 @@ jobs: python -m pip install --no-build-isolation .[all,testing] python -m pip list shell: bash - - name: Run compiled tests + - name: Run quick tests run: | - python -m pip uninstall -y monai - BUILD_MONAI=1 python -m pip install --no-build-isolation -e . # compile the cpp extensions python -c 'import monai._C' > /dev/null nvidia-smi || true python monai/config/check_env.py --env --monai - python -m unittest -v + # python -m unittest -v + ./runtests.sh --min + env: + QUICKTEST: True + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + NGC_ORG: ${{ secrets.NGC_ORG }} + NGC_TEAM: ${{ secrets.NGC_TEAM }} diff --git a/monai/engines/trainer.py b/monai/engines/trainer.py index 1f0c75620f..df1853f7b1 100644 --- a/monai/engines/trainer.py +++ b/monai/engines/trainer.py @@ -51,7 +51,7 @@ def run(self) -> None: # type: ignore[override] If call this function multiple times, it will continuously run from the previous state. """ - self.scaler = torch.cuda.amp.GradScaler() if self.amp else None + self.scaler = torch.amp.GradScaler() if self.amp else None super().run() def get_stats(self, *vars): @@ -613,8 +613,8 @@ def __init__( self.g_inferer = SimpleInferer() if g_inferer is None else g_inferer self.d_inferer = SimpleInferer() if d_inferer is None else d_inferer - self.state.g_scaler = torch.cuda.amp.GradScaler() if self.amp else None - self.state.d_scaler = torch.cuda.amp.GradScaler() if self.amp else None + self.state.g_scaler = torch.amp.GradScaler() if self.amp else None + self.state.d_scaler = torch.amp.GradScaler() if self.amp else None self.optim_set_to_none = optim_set_to_none self._complete_state_dict_user_keys() diff --git a/monai/engines/workflow.py b/monai/engines/workflow.py index ecb0c4a070..95a85eecb3 100644 --- a/monai/engines/workflow.py +++ b/monai/engines/workflow.py @@ -165,7 +165,7 @@ def set_sampler_epoch(engine: Engine) -> None: self.amp = amp self.to_kwargs = {} if to_kwargs is None else to_kwargs self.amp_kwargs = {} if amp_kwargs is None else amp_kwargs - self.scaler: torch.cuda.amp.GradScaler | None = None + self.scaler: torch.amp.GradScaler | None = None if event_names is None: event_names = [IterationEvents] diff --git a/tests/integration/test_integration_fast_train.py b/tests/integration/test_integration_fast_train.py index 814c4b182c..651264f7e5 100644 --- a/tests/integration/test_integration_fast_train.py +++ b/tests/integration/test_integration_fast_train.py @@ -165,7 +165,7 @@ def test_train_timing(self): # Novograd paper suggests to use a bigger LR than Adam, # because Adam does normalization by element-wise second moments optimizer = Novograd(model.parameters(), learning_rate * 10) - scaler = torch.cuda.amp.GradScaler() + scaler = torch.amp.GradScaler() post_pred = AsDiscrete(argmax=True, to_onehot=2) post_label = AsDiscrete(to_onehot=2) From c6f79f44dd4904b0c70197bfd0dec682de342a83 Mon Sep 17 00:00:00 2001 From: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> Date: Sun, 25 Oct 2026 11:20:00 +0000 Subject: [PATCH 6/9] BUILD_MONAI? Signed-off-by: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> --- .github/workflows/cicd_tests_gpu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cicd_tests_gpu.yml b/.github/workflows/cicd_tests_gpu.yml index 04cdb9ccdd..e411a13f03 100644 --- a/.github/workflows/cicd_tests_gpu.yml +++ b/.github/workflows/cicd_tests_gpu.yml @@ -74,6 +74,7 @@ jobs: python -m pip install --pre -U itk python -m pip install --user --upgrade pip wheel tomli python monai/config/print_dependencies.py build-system | xargs pip install --no-build-isolation + echo "BUILD_MONAI=$BUILD_MONAI" python -m pip install --no-build-isolation .[all,testing] python -m pip list shell: bash From dbad3b927b9303f0483502e2e5117500b016d1c8 Mon Sep 17 00:00:00 2001 From: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> Date: Sun, 25 Oct 2026 11:31:02 +0000 Subject: [PATCH 7/9] BUILD_MONAI? Signed-off-by: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> --- .github/workflows/cicd_tests_gpu.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cicd_tests_gpu.yml b/.github/workflows/cicd_tests_gpu.yml index e411a13f03..d93e382a0a 100644 --- a/.github/workflows/cicd_tests_gpu.yml +++ b/.github/workflows/cicd_tests_gpu.yml @@ -74,8 +74,7 @@ jobs: python -m pip install --pre -U itk python -m pip install --user --upgrade pip wheel tomli python monai/config/print_dependencies.py build-system | xargs pip install --no-build-isolation - echo "BUILD_MONAI=$BUILD_MONAI" - python -m pip install --no-build-isolation .[all,testing] + BUILD_MONAI=1 python -m pip install --no-build-isolation .[all,testing] python -m pip list shell: bash - name: Run quick tests From 10d8f541d0e6033d8668ce28edaf204254e58631 Mon Sep 17 00:00:00 2001 From: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> Date: Sun, 25 Oct 2026 11:45:14 +0000 Subject: [PATCH 8/9] BUILD_MONAI? Signed-off-by: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> --- .github/workflows/cicd_tests_gpu.yml | 2 +- .gitignore | 2 ++ pyproject.toml | 6 ++---- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cicd_tests_gpu.yml b/.github/workflows/cicd_tests_gpu.yml index d93e382a0a..e0d9dedab1 100644 --- a/.github/workflows/cicd_tests_gpu.yml +++ b/.github/workflows/cicd_tests_gpu.yml @@ -74,7 +74,7 @@ jobs: python -m pip install --pre -U itk python -m pip install --user --upgrade pip wheel tomli python monai/config/print_dependencies.py build-system | xargs pip install --no-build-isolation - BUILD_MONAI=1 python -m pip install --no-build-isolation .[all,testing] + python -m pip install --no-build-isolation -e .[all,testing] python -m pip list shell: bash - name: Run quick tests diff --git a/.gitignore b/.gitignore index d0bdc54018..c22d2c9802 100644 --- a/.gitignore +++ b/.gitignore @@ -170,3 +170,5 @@ runs *.pth *zarr/* + +uv.lock diff --git a/pyproject.toml b/pyproject.toml index aa434b364f..3c748a67cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,8 @@ name = "monai" description = "AI Toolkit for Healthcare Imaging" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.10" -license = { text = "Apache License 2.0" } +license = "Apache License 2.0" +license-files = ["LICENSE"] authors = [{ name = "MONAI Consortium", email = "monai.contact@gmail.com" }] classifiers = [ "Intended Audience :: Developers", @@ -186,9 +187,6 @@ testing = [ "types-setuptools" ] -[tool.setuptools] -license-files = ["LICENSE"] - [tool.setuptools.dynamic] version = {attr = "monai.__version__"} From 460f12d6af73686bf93f80d9207af3c91faaf849 Mon Sep 17 00:00:00 2001 From: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:06:22 +0000 Subject: [PATCH 9/9] License fix Signed-off-by: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3c748a67cb..8813d76fd4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,6 @@ name = "monai" description = "AI Toolkit for Healthcare Imaging" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.10" -license = "Apache License 2.0" license-files = ["LICENSE"] authors = [{ name = "MONAI Consortium", email = "monai.contact@gmail.com" }] classifiers = [