From f6d51182c9222ef0755cdf881e6f8727c7b2ea63 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 20 Oct 2026 13:39:53 -0500 Subject: [PATCH 1/8] initial gpt-guided implementation of tasks sharing gpus (getting assigned logical "shares" of physical gpus). libE_specs["gpu_sharing_factor"] requests number of logical shares to make out of each physical gpu. gpu_sharing_factor of 1 (default) produces traditional behavior. shares assigned round-robin. workers convert assigned shares into physical GPU ids. associated tests. --- libensemble/executors/mpi_runner.py | 8 ++-- libensemble/resources/resources.py | 1 + libensemble/resources/rset_resources.py | 38 ++++++++++++--- libensemble/resources/worker_resources.py | 32 ++++++++++++- libensemble/specs.py | 12 +++++ libensemble/tests/unit_tests/test_models.py | 8 ++++ .../tests/unit_tests/test_resources.py | 46 ++++++++++++++++++- libensemble/tools/test_support.py | 6 +-- 8 files changed, 134 insertions(+), 17 deletions(-) diff --git a/libensemble/executors/mpi_runner.py b/libensemble/executors/mpi_runner.py index d44152aff..015430038 100644 --- a/libensemble/executors/mpi_runner.py +++ b/libensemble/executors/mpi_runner.py @@ -124,8 +124,8 @@ def _set_gpu_cli_option(self, wresources, extra_args, gpu_setting_name, gpu_valu def _set_gpu_env_var(self, wresources, task, gpus_per_node, gpus_env): """Add GPU environment variable setting to the tasks environment""" jassert(wresources.matching_slots, f"Cannot assign CPUs/GPUs to non-matching slots per node {wresources.slots}") - slot_list = wresources.get_slots_as_string(multiplier=wresources.gpus_per_rset_per_node, limit=gpus_per_node) - task._add_to_env(gpus_env, slot_list) + gpu_list = wresources.get_gpu_ids_as_string(limit=gpus_per_node) + task._add_to_env(gpus_env, gpu_list) def _local_runner_set_gpus(self, task, wresources, extra_args, gpus_per_node, ppn): """Set default GPU setting for MPI runner""" @@ -174,7 +174,7 @@ def _assign_gpus(self, task, resources, nprocs, nnodes, ppn, ngpus, extra_args, # gpus per node for this worker. if wresources.doihave_gpus(): - gpus_avail_per_node = wresources.slot_count * wresources.gpus_per_rset_per_node + gpus_avail_per_node = len(set(wresources.gpu_ids_on_node)) else: gpus_avail_per_node = 0 @@ -246,7 +246,7 @@ def _get_min_nodes(self, nprocs, ppn, nnodes, ngpus, resources): if nprocs: proc_min_nodes = (nprocs + procs_on_node - 1) // procs_on_node if ngpus: - gpus_on_node = wresources.slot_count * wresources.gpus_per_rset_per_node + gpus_on_node = len(set(wresources.gpu_ids_on_node)) gpu_min_nodes = (ngpus + gpus_on_node - 1) // gpus_on_node min_nodes = max(proc_min_nodes, gpu_min_nodes) diff --git a/libensemble/resources/resources.py b/libensemble/resources/resources.py index 9b68e43f0..7bd7e008e 100644 --- a/libensemble/resources/resources.py +++ b/libensemble/resources/resources.py @@ -164,6 +164,7 @@ def __init__(self, libE_specs: dict, platform_info: dict = {}, top_level_dir: st self.num_resource_sets = libE_specs.get("num_resource_sets", None) self.enforce_worker_core_bounds = libE_specs.get("enforce_worker_core_bounds", False) self.gpus_per_group = libE_specs.get("gpus_per_group") + self.gpu_sharing_factor = libE_specs.get("gpu_sharing_factor", 1) resource_info = libE_specs.get("resource_info", {}) # resource_info overrides platform diff --git a/libensemble/resources/rset_resources.py b/libensemble/resources/rset_resources.py index e43a51b88..6278b8dda 100644 --- a/libensemble/resources/rset_resources.py +++ b/libensemble/resources/rset_resources.py @@ -33,6 +33,8 @@ class RSetResources: ("group", int), # Group ID this resource set belongs to ("slot", int), # Slot ID this resource set belongs to ("gpus", bool), # Does this resource set have GPUs + ("gpu_id", int), # Physical GPU ID for GPU resource sets + ("gpu_share", int), # Logical share within a physical GPU # ('pool', int), # Pool ID (eg. separate gen/sim resources) - not yet used. ] @@ -61,19 +63,31 @@ def __init__(self, num_workers: int, resources: GlobalResources): self.nodes_in_rset = len(self.split_list[0]) gpus_avail_per_node = resources.gpus_avail_per_node + self.gpu_sharing_factor = resources.gpu_sharing_factor self.rsets_per_node = RSetResources.get_rsets_on_a_node(self.total_num_rsets, resources) - self.gpu_rsets_per_node = min(gpus_avail_per_node, self.rsets_per_node) + self.gpu_rsets_per_node = min(gpus_avail_per_node * self.gpu_sharing_factor, self.rsets_per_node) self.nongpu_rsets_per_node = self.rsets_per_node - self.gpu_rsets_per_node self.all_rsets = np.zeros(self.total_num_rsets, dtype=RSetResources.rset_dtype) self.all_rsets["group"], self.all_rsets["slot"], self.all_rsets["gpus"] = RSetResources.get_group_list( - self.split_list, gpus_avail_per_node, resources.gpus_per_group + self.split_list, gpus_avail_per_node, resources.gpus_per_group, self.gpu_sharing_factor ) + if gpus_avail_per_node: + gpu_positions = RSetResources.get_gpu_positions(self.split_list) + self.all_rsets["gpu_id"] = np.where(self.all_rsets["gpus"], gpu_positions % gpus_avail_per_node, -1) + self.all_rsets["gpu_share"] = np.where(self.all_rsets["gpus"], gpu_positions // gpus_avail_per_node, -1) self.total_num_gpu_rsets = np.count_nonzero(self.all_rsets["gpus"]) self.total_num_nongpu_rsets = np.count_nonzero(~self.all_rsets["gpus"]) - self.gpus_per_rset_per_node = gpus_avail_per_node // self.gpu_rsets_per_node if self.gpu_rsets_per_node else 0 + if self.gpu_rsets_per_node: + self.gpus_per_rset_per_node = ( + 1 + if self.gpu_sharing_factor > 1 + else gpus_avail_per_node // min(gpus_avail_per_node, self.rsets_per_node) + ) + else: + self.gpus_per_rset_per_node = 0 self.cores_per_rset_per_node = resources.physical_cores_avail_per_node // self.rsets_per_node # Oversubsribe @@ -89,8 +103,8 @@ def __init__(self, num_workers: int, resources: GlobalResources): self.procs_per_rset = self.procs_per_rset_per_node * self.nodes_in_rset @staticmethod - def get_group_list(split_list, gpus_per_node=0, gpus_per_group=None): - """Return lists of group ids and slot IDs by resource set""" + def get_group_list(split_list, gpus_per_node=0, gpus_per_group=None, gpu_sharing_factor=1): + """Return group, slot, and GPU lists by resource set""" group = 1 slot = 0 group_list, slot_list, gpu_list = [], [], [] @@ -104,10 +118,22 @@ def get_group_list(split_list, gpus_per_node=0, gpus_per_group=None): slot = 0 group_list.append(group) slot_list.append(slot) - gpu_list.append(slot < gpus_per_node) + is_gpu = slot < gpus_per_node * gpu_sharing_factor + gpu_list.append(is_gpu) slot += 1 return group_list, slot_list, gpu_list + @staticmethod + def get_gpu_positions(split_list): + """Return each resource set's node-local position, independent of scheduler groups.""" + positions = [] + node_counts = {} + for nodes in split_list: + node = nodes[0] + positions.append(node_counts.get(node, 0)) + node_counts[node] = positions[-1] + 1 + return np.asarray(positions) + @staticmethod def best_split(a, n): """Creates the most even split of list a into n parts and return list of lists""" diff --git a/libensemble/resources/worker_resources.py b/libensemble/resources/worker_resources.py index 0207b9089..dff510570 100644 --- a/libensemble/resources/worker_resources.py +++ b/libensemble/resources/worker_resources.py @@ -198,6 +198,7 @@ def __init__(self, num_workers, resources, workerID): self.matching_slots = True self.slot_count = None self.slots_on_node = None + self.gpu_ids_on_node = None self.local_node_count = len(self.local_nodelist) self.set_slot_count() self.gen_nprocs = None @@ -252,6 +253,23 @@ def set_env_to_slots(self, env_var, multiplier=1, delimiter=","): """ os.environ[env_var] = self.get_slots_as_string(multiplier, delimiter) + def get_gpu_ids_as_string(self, delimiter=",", limit=None): + """Returns physical GPU IDs for the worker's assigned GPU resource sets.""" + if self.gpu_ids_on_node is None: + logger.warning("GPU IDs on node are None when requested as a string") + return None + gpu_ids = [] + for gpu_id in self.gpu_ids_on_node: + if gpu_id >= 0: + gpu_ids.extend(range(gpu_id * self.gpus_per_rset_per_node, (gpu_id + 1) * self.gpus_per_rset_per_node)) + gpu_ids = list(OrderedDict.fromkeys(gpu_ids)) + if limit is not None: + gpu_ids = gpu_ids[:limit] + if self.tiles_per_gpu > 1: + ntiles = self.tiles_per_gpu + gpu_ids = [f"{i // ntiles}.{i % ntiles}" for i in gpu_ids] + return delimiter.join(map(str, gpu_ids)) + def set_env_to_gpus(self, env_var=None, delimiter=","): """Sets the given environment variable to GPUs @@ -276,7 +294,7 @@ def set_env_to_gpus(self, env_var=None, delimiter=","): """ assert self.matching_slots, f"Cannot assign GPUs to non-matching slots per node {self.slots}" if self.doihave_gpus(): - env_value = self.get_slots_as_string(multiplier=self.gpus_per_rset_per_node, limit=self.gen_ngpus) + env_value = self.get_gpu_ids_as_string(limit=self.gen_ngpus) if env_var is None: if self.platform_info is not None: if self.platform_info.get("gpu_setting_type") == "env": @@ -315,6 +333,9 @@ def set_rset_team(self, rset_team: list[int]) -> None: self.split_list, self.rsets_per_node, ) + self.gpu_ids_on_node = WorkerResources.get_local_gpu_ids( + self.rset_team, self.all_rsets, self.split_list, self.local_nodelist[0] if self.local_nodelist else None + ) self.set_slot_count() self.local_node_count = len(self.local_nodelist) @@ -346,6 +367,15 @@ def set_slot_count(self) -> None: self.slots_on_node = first_node_slots if self.matching_slots else None self.slot_count = first_len if self.even_slots else None + @staticmethod + def get_local_gpu_ids(rset_team: list[int], all_rsets, split_list, node) -> list[int]: + """Return physical GPU IDs on the worker's first local node.""" + return [ + int(all_rsets["gpu_id"][rset]) + for rset in rset_team + if all_rsets["gpus"][rset] and (node is None or split_list[rset][0] == node) + ] + @staticmethod def get_local_nodelist( workerID: int, rset_team: list[int], split_list: list[list[str]], rsets_per_node: int diff --git a/libensemble/specs.py b/libensemble/specs.py index d87030829..fe8abdaef 100644 --- a/libensemble/specs.py +++ b/libensemble/specs.py @@ -778,6 +778,18 @@ def set_calc_dirs_on_input_dir(self): If not set, resources will be divided evenly by the number of workers. """ + gpu_sharing_factor: int = 1 + """ + Number of logical GPU shares available for each physical GPU. + A value of 1 preserves exclusive GPU allocation. + """ + + @field_validator("gpu_sharing_factor") + def check_gpu_sharing_factor(cls, value): + if value < 1: + raise ValueError("gpu_sharing_factor must be at least 1") + return value + gen_num_procs: int | None = 0 """ The default number of processors (MPI ranks) required by generators. Unless diff --git a/libensemble/tests/unit_tests/test_models.py b/libensemble/tests/unit_tests/test_models.py index 3dd3ba3cd..fcb6c4655 100644 --- a/libensemble/tests/unit_tests/test_models.py +++ b/libensemble/tests/unit_tests/test_models.py @@ -1,4 +1,5 @@ import numpy as np +import pytest from gest_api.vocs import VOCS from pydantic import ValidationError @@ -96,6 +97,13 @@ def test_libe_specs_invalid(): assert flag, "LibeSpecs didn't raise ValidationError on invalid specs" +def test_gpu_sharing_factor_validation(): + assert LibeSpecs().gpu_sharing_factor == 1 + assert LibeSpecs(gpu_sharing_factor=2).gpu_sharing_factor == 2 + with pytest.raises(ValueError, match="gpu_sharing_factor must be at least 1"): + LibeSpecs(gpu_sharing_factor=0) + + def test_ensemble_specs(): sim_specs, gen_specs, exit_criteria = setup.make_criteria_and_specs_0() libE_specs = {"comms": "local", "nworkers": 4} diff --git a/libensemble/tests/unit_tests/test_resources.py b/libensemble/tests/unit_tests/test_resources.py index b79a5eeff..e2fd9ca43 100644 --- a/libensemble/tests/unit_tests/test_resources.py +++ b/libensemble/tests/unit_tests/test_resources.py @@ -1,6 +1,7 @@ import os import socket +from libensemble.executors.mpi_runner import MPICH_MPIRunner from libensemble.resources.env_resources import EnvResources from libensemble.resources.mpi_resources import create_machinefile from libensemble.resources.resources import GlobalResources, Resources, ResourcesException @@ -780,12 +781,19 @@ def test_machinefile_from_resources(): os.remove("machinefile") -def _setup_wresources_gpus(ngpus): +def _setup_wresources_gpus(ngpus, gpu_sharing_factor=1, num_resource_sets=None, gpus_per_group=None): """Set up worker resources with a given number of GPUs""" os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "knl-[0020,0036]" resource_info = {"nodelist_env_slurm": "LIBE_RESOURCES_TEST_NODE_LIST", "gpus_on_node": ngpus} - libE_specs = {"resource_info": resource_info, "dedicated_mode": False} + libE_specs = { + "resource_info": resource_info, + "dedicated_mode": False, + "gpu_sharing_factor": gpu_sharing_factor, + "gpus_per_group": gpus_per_group, + } + if num_resource_sets is not None: + libE_specs["num_resource_sets"] = num_resource_sets gresources = GlobalResources(libE_specs) nworkers = 8 workerID = 2 # Does not matter as using set_rset_team @@ -915,6 +923,40 @@ def test_wresources_set_gpus_by_platform_gpu_tiles(): del wresources +def test_wresources_set_gpus_with_sharing(): + wresources = _setup_wresources_gpus(4, gpu_sharing_factor=2, num_resource_sets=16) + + assert wresources.total_num_gpu_rsets == 16 + assert list(wresources.all_rsets["gpu_id"]) == [0, 1, 2, 3, 0, 1, 2, 3] * 2 + assert list(wresources.all_rsets["gpu_share"]) == ([0] * 4 + [1] * 4) * 2 + + wresources.set_rset_team([0, 1, 4, 5]) + wresources.set_env_to_gpus("LIBE_TEST_GPUS_1") + assert os.environ["LIBE_TEST_GPUS_1"] == "0,1" + os.environ.pop("LIBE_TEST_GPUS_1") + + +def test_gpu_sharing_executor_env_mapping(): + wresources = _setup_wresources_gpus(4, gpu_sharing_factor=2, num_resource_sets=16) + wresources.set_rset_team([4]) + + class Task: + def _add_to_env(self, name, value): + self.env = {name: value} + + task = Task() + MPICH_MPIRunner()._set_gpu_env_var(wresources, task, 1, "CUDA_VISIBLE_DEVICES") + assert task.env == {"CUDA_VISIBLE_DEVICES": "0"} + + +def test_gpu_sharing_mapping_with_gpu_groups(): + wresources = _setup_wresources_gpus(4, gpu_sharing_factor=2, num_resource_sets=16, gpus_per_group=2) + + assert list(wresources.all_rsets["slot"]) == [0, 1] * 8 + assert list(wresources.all_rsets["gpu_id"]) == [0, 1, 2, 3, 0, 1, 2, 3] * 2 + assert list(wresources.all_rsets["gpu_share"]) == ([0] * 4 + [1] * 4) * 2 + + def test_wresources_set_gpus_x2(): wresources = _setup_wresources_gpus(8) diff --git a/libensemble/tools/test_support.py b/libensemble/tools/test_support.py index 7e0727125..4a5b41657 100644 --- a/libensemble/tools/test_support.py +++ b/libensemble/tools/test_support.py @@ -204,7 +204,7 @@ def check_gpu_setting(task, assert_setting=True, print_setting=False, resources= # Get expected numbers if cmd_line: - expected_nums = _safe_min(wresources.slot_count * wresources.gpus_per_rset_per_node, wresources.gen_ngpus) + expected_nums = _safe_min(len(set(wresources.gpu_ids_on_node)), wresources.gen_ngpus) if gpus_per_task: stype = "runline option: gpus per task" expected_nums //= int(ppn) @@ -219,9 +219,7 @@ def check_gpu_setting(task, assert_setting=True, print_setting=False, resources= gpu_setting = _get_opt_value(expected_setting, task.runline) else: stype = "Env var" - expected_nums = wresources.get_slots_as_string( - multiplier=wresources.gpus_per_rset_per_node, limit=wresources.gen_ngpus - ) + expected_nums = wresources.get_gpu_ids_as_string(limit=wresources.gen_ngpus) expected_nums = expected_nums if _set_gpus(task, wresources) else None if expected_nums is not None: expected = {expected_setting: expected_nums} From 44f9eafe39152dad7e225329c2cf1cc90f5e95a4 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 27 Oct 2026 13:55:44 -0500 Subject: [PATCH 2/8] H/libE_info can display fractional values to describe how much of a GPU was assigned. new "pack" policy for scheduling fractional gpu shares (multiple jobs get packed onto a GPU by default). "pack" or "spread" options for gpu_packing. scheduler builds "gpu_bins" for allocating into. jobs can still request a whole GPU. tests. --- libensemble/executors/mpi_runner.py | 14 +- libensemble/resources/rset_resources.py | 13 +- libensemble/resources/scheduler.py | 167 ++++++++++++++++++ libensemble/resources/worker_resources.py | 6 + libensemble/specs.py | 3 +- .../tests/unit_tests/test_resources.py | 14 +- libensemble/tools/alloc_support.py | 69 ++++++-- 7 files changed, 257 insertions(+), 29 deletions(-) diff --git a/libensemble/executors/mpi_runner.py b/libensemble/executors/mpi_runner.py index 015430038..c79b6aee3 100644 --- a/libensemble/executors/mpi_runner.py +++ b/libensemble/executors/mpi_runner.py @@ -188,7 +188,12 @@ def _assign_gpus(self, task, resources, nprocs, nnodes, ppn, ngpus, extra_args, nnodes = wresources.local_node_count if ngpus is not None: - gpus_req_per_node = ngpus // nnodes + import math + + try: + gpus_req_per_node = math.ceil(float(ngpus) / nnodes) + except Exception: + gpus_req_per_node = ngpus // nnodes if gpus_req_per_node > gpus_avail_per_node: logger.info(f"Asked for more GPUs per node than available - max is {gpus_avail_per_node}") gpus_per_node = min(gpus_req_per_node, gpus_avail_per_node) @@ -247,7 +252,12 @@ def _get_min_nodes(self, nprocs, ppn, nnodes, ngpus, resources): proc_min_nodes = (nprocs + procs_on_node - 1) // procs_on_node if ngpus: gpus_on_node = len(set(wresources.gpu_ids_on_node)) - gpu_min_nodes = (ngpus + gpus_on_node - 1) // gpus_on_node + import math + + try: + gpu_min_nodes = math.ceil(float(ngpus) / gpus_on_node) if gpus_on_node else 1 + except Exception: + gpu_min_nodes = (ngpus + gpus_on_node - 1) // gpus_on_node min_nodes = max(proc_min_nodes, gpu_min_nodes) nnodes = min(min_nodes, total_nodes) diff --git a/libensemble/resources/rset_resources.py b/libensemble/resources/rset_resources.py index 6278b8dda..16a6f5953 100644 --- a/libensemble/resources/rset_resources.py +++ b/libensemble/resources/rset_resources.py @@ -74,8 +74,12 @@ def __init__(self, num_workers: int, resources: GlobalResources): ) if gpus_avail_per_node: gpu_positions = RSetResources.get_gpu_positions(self.split_list) - self.all_rsets["gpu_id"] = np.where(self.all_rsets["gpus"], gpu_positions % gpus_avail_per_node, -1) - self.all_rsets["gpu_share"] = np.where(self.all_rsets["gpus"], gpu_positions // gpus_avail_per_node, -1) + self.all_rsets["gpu_id"] = np.where( + self.all_rsets["gpus"], + (gpu_positions // self.gpu_sharing_factor) % gpus_avail_per_node, + -1, + ) + self.all_rsets["gpu_share"] = np.where(self.all_rsets["gpus"], gpu_positions % self.gpu_sharing_factor, -1) self.total_num_gpu_rsets = np.count_nonzero(self.all_rsets["gpus"]) self.total_num_nongpu_rsets = np.count_nonzero(~self.all_rsets["gpus"]) @@ -111,8 +115,9 @@ def get_group_list(split_list, gpus_per_node=0, gpus_per_group=None, gpu_sharing node = split_list[0] for i in range(len(split_list)): - # still break on new node if gpus_per_group is set - if split_list[i] != node or slot == gpus_per_group: + # break on new node or when group size reached (account for sharing) + group_threshold = gpus_per_group * gpu_sharing_factor if gpus_per_group is not None else None + if split_list[i] != node or slot == group_threshold: node = split_list[i] group += 1 slot = 0 diff --git a/libensemble/resources/scheduler.py b/libensemble/resources/scheduler.py index 2e75e3c84..860099737 100644 --- a/libensemble/resources/scheduler.py +++ b/libensemble/resources/scheduler.py @@ -57,6 +57,13 @@ def __init__(self, user_resources=None, sched_opts={}): Useful if setting an environment variable such as ``CUDA_VISIBLE_DEVICES`` to specific slots, which should match over multiple nodes. Default: True + + "gpu_packing" [String]: + GPU share packing strategy when ``gpu_sharing_factor > 1``. + "pack" (default) co-locates fractional tasks on the most-occupied GPU + to preserve fully-free GPUs for exclusive tasks. "spread" distributes + across least-occupied GPUs. + Default: "pack" """ self.resources = user_resources or Resources.resources.resource_manager @@ -71,6 +78,9 @@ def __init__(self, user_resources=None, sched_opts={}): # Process scheduler options self.split2fit = sched_opts.get("split2fit", True) self.match_slots = sched_opts.get("match_slots", True) + self.gpu_packing = sched_opts.get("gpu_packing", "pack") + if self.gpu_packing not in ("pack", "spread"): + raise ValueError(f"gpu_packing must be 'pack' or 'spread', got {self.gpu_packing!r}") self.last_use_gpus = None def assign_resources(self, rsets_req, use_gpus=None, user_params=[]): @@ -92,6 +102,10 @@ def assign_resources(self, rsets_req, use_gpus=None, user_params=[]): return [] self.check_total_rsets(rsets_req, use_gpus) + + # GPU-share-aware path for physical-exclusive packing + if use_gpus and getattr(self.resources, "gpu_sharing_factor", 1) > 1: + return self._assign_gpu_sharing_resources(rsets_req, use_gpus, user_params) self.log_msg = None # Log resource messages only when find resources num_groups = self.resources.num_groups @@ -180,6 +194,159 @@ def assign_resources(self, rsets_req, use_gpus=None, user_params=[]): return rset_team + def _build_gpu_bins(self, valid_rsets_by_group): + """Build per-group GPU bins: group -> gpu_id -> list[rset_id]""" + gpu_bins_per_group = {} + for g, rlist in valid_rsets_by_group.items(): + bins = {} + for rid in rlist: + gid = int(self.resources.rsets[rid]["gpu_id"]) + bins.setdefault(gid, []).append(rid) + for gid in bins: + bins[gid] = sorted(bins[gid]) + gpu_bins_per_group[g] = bins + return gpu_bins_per_group + + def _can_group_satisfy_gpu(self, group, need, gpu_bins_per_group): + """Check if group can satisfy need with physical-exclusive semantics""" + bins = gpu_bins_per_group.get(group, {}) + sf = getattr(self.resources, "gpu_sharing_factor", 1) + whole = need // sf + rem = need % sf + fully_free = sum(1 for lst in bins.values() if len(lst) == sf) + if fully_free < whole: + return False + if rem == 0: + return True + leftover_fully = fully_free - whole + if leftover_fully > 0: + return True + for lst in bins.values(): + if len(lst) == sf: + continue + if len(lst) >= rem: + return True + total_free = sum(len(lst) for lst in bins.values()) + return (total_free - whole * sf) >= rem + + def _select_rsets_for_gpu_group(self, group, need, gpu_bins_per_group): + """Select rsets for one group using pack/spread""" + bins = gpu_bins_per_group[group] + sf = getattr(self.resources, "gpu_sharing_factor", 1) + whole = need // sf + rem = need % sf + selected = [] + fully_free_gids = sorted([gid for gid, lst in bins.items() if len(lst) == sf]) + if len(fully_free_gids) < whole: + raise InsufficientFreeResources + chosen_whole = fully_free_gids[:whole] + for gid in chosen_whole: + selected.extend(bins[gid]) + remaining_bins = {gid: lst for gid, lst in bins.items() if gid not in chosen_whole} + if rem > 0: + candidates = [(len(lst), gid) for gid, lst in remaining_bins.items() if len(lst) >= rem] + if candidates: + if self.gpu_packing == "pack": + candidates.sort(key=lambda x: (x[0], x[1])) + else: + candidates.sort(key=lambda x: (-x[0], x[1])) + chosen_gid = candidates[0][1] + selected.extend(remaining_bins[chosen_gid][:rem]) + else: + sorted_bins = [(len(lst), gid, lst) for gid, lst in remaining_bins.items()] + if self.gpu_packing == "pack": + sorted_bins.sort(key=lambda x: (x[0], x[1])) + else: + sorted_bins.sort(key=lambda x: (-x[0], x[1])) + need_rem = rem + for _, gid, lst in sorted_bins: + take = min(len(lst), need_rem) + selected.extend(lst[:take]) + need_rem -= take + if need_rem == 0: + break + if need_rem > 0: + raise InsufficientFreeResources + return sorted(selected) + + def _assign_gpu_sharing_resources(self, rsets_req, use_gpus, user_params): + """Physical-exclusive GPU-share-aware assignment with packing""" + num_groups = self.resources.num_groups + max_grpsize = self.resources.gpu_rsets_per_node + avail_rsets_by_group = self.get_avail_rsets_by_group() + valid_rsets_by_group = self.filter_for_rset_type(avail_rsets_by_group, use_gpus) + rsets_req, num_groups_req, rsets_per_group = self.calc_req_split( + rsets_req, max_grpsize, num_groups, extend=True + ) + sorted_lengths = ResourceScheduler.get_sorted_lens(valid_rsets_by_group) + if num_groups_req > len(sorted_lengths) or sorted_lengths[num_groups_req - 1] < rsets_per_group: + if not self.split2fit or (sorted_lengths and sorted_lengths[0] == 0): + raise InsufficientFreeResources + gpu_bins_per_group = self._build_gpu_bins(valid_rsets_by_group) + + def feasible_groups_for(need_per_group, ngroups): + feasible = [ + g for g in valid_rsets_by_group if self._can_group_satisfy_gpu(g, need_per_group, gpu_bins_per_group) + ] + + # sort by packing preference: pack prefers most occupied (smallest free) that still feasible + def sort_key(g): + total_free = len(valid_rsets_by_group[g]) + return (total_free, g) if self.gpu_packing == "pack" else (-total_free, g) + + feasible.sort(key=sort_key) + return feasible + + feasible = feasible_groups_for(rsets_per_group, num_groups_req) + if len(feasible) < num_groups_req: + if not self.split2fit: + raise InsufficientFreeResources + # Try to find alternative split that is feasible + sorted_lengths = ResourceScheduler.get_sorted_lens(valid_rsets_by_group) + max_even = sorted_lengths[num_groups_req - 1] if num_groups_req <= len(sorted_lengths) else 0 + tried = set() + while len(feasible) < num_groups_req: + key = (rsets_req, num_groups_req, rsets_per_group) + if key in tried: + num_groups_req += 1 + if num_groups_req > num_groups: + raise InsufficientFreeResources + rsets_per_group = sorted_lengths[num_groups_req - 1] if num_groups_req <= len(sorted_lengths) else 0 + tried = set() + continue + tried.add(key) + try: + rsets_req, num_groups_req, rsets_per_group = self.calc_even_split_uneven_groups( + max_even, num_groups_req, rsets_req, sorted_lengths, num_groups, user_params + ) + except InsufficientFreeResources: + raise + feasible = feasible_groups_for(rsets_per_group, num_groups_req) + if len(feasible) >= num_groups_req: + break + # if still not feasible, try larger group count + if rsets_per_group * num_groups_req != rsets_req or not self.check_params(user_params, num_groups_req): + continue + # fallback: increment groups + if len(feasible) < num_groups_req: + max_even = sorted_lengths[num_groups_req - 1] if num_groups_req <= len(sorted_lengths) else 0 + if len(feasible) < num_groups_req: + raise InsufficientFreeResources + cand_groups = feasible[:num_groups_req] + rset_team = [] + for g in cand_groups: + part = self._select_rsets_for_gpu_group(g, rsets_per_group, gpu_bins_per_group) + rset_team.extend(part) + rset_team = sorted(rset_team) + self.avail_rsets_by_group = self.filter_out_rset_team(avail_rsets_by_group, rset_team) + self.rsets_free -= len(rset_team) + self.gpu_rsets_free -= len(rset_team) + logger.debug( + f"rset_team found (gpu-sharing): Req: {rsets_req} rsets. Found: {rset_team}" + + f" Avail sets {self.avail_rsets_by_group}" + ) + return rset_team + def find_rsets_any_slots(self, valid_rsets_by_group, max_grpsize, rsets_req, ngroups, rsets_per_group): """Find optimal non-matching slots across groups""" tmp_rsets_by_group = copy.deepcopy(valid_rsets_by_group) diff --git a/libensemble/resources/worker_resources.py b/libensemble/resources/worker_resources.py index dff510570..b6e5ba9e0 100644 --- a/libensemble/resources/worker_resources.py +++ b/libensemble/resources/worker_resources.py @@ -264,6 +264,12 @@ def get_gpu_ids_as_string(self, delimiter=",", limit=None): gpu_ids.extend(range(gpu_id * self.gpus_per_rset_per_node, (gpu_id + 1) * self.gpus_per_rset_per_node)) gpu_ids = list(OrderedDict.fromkeys(gpu_ids)) if limit is not None: + try: + import math + + limit = math.ceil(float(limit)) + except Exception: + pass gpu_ids = gpu_ids[:limit] if self.tiles_per_gpu > 1: ntiles = self.tiles_per_gpu diff --git a/libensemble/specs.py b/libensemble/specs.py index fe8abdaef..a0ce2fc5c 100644 --- a/libensemble/specs.py +++ b/libensemble/specs.py @@ -797,11 +797,12 @@ def check_gpu_sharing_factor(cls, value): allocated this many processors for applications launched via the MPIExecutor. """ - gen_num_gpus: int | None = 0 + gen_num_gpus: int | float | None = 0 """ The default number of GPUs required by generators. Unless overridden by the equivalent `persis_info` settings, generators will be allocated this many GPUs. + Supports fractional values when ``gpu_sharing_factor > 1``. """ gpus_per_group: int | None = None diff --git a/libensemble/tests/unit_tests/test_resources.py b/libensemble/tests/unit_tests/test_resources.py index e2fd9ca43..fb88a60f3 100644 --- a/libensemble/tests/unit_tests/test_resources.py +++ b/libensemble/tests/unit_tests/test_resources.py @@ -927,10 +927,10 @@ def test_wresources_set_gpus_with_sharing(): wresources = _setup_wresources_gpus(4, gpu_sharing_factor=2, num_resource_sets=16) assert wresources.total_num_gpu_rsets == 16 - assert list(wresources.all_rsets["gpu_id"]) == [0, 1, 2, 3, 0, 1, 2, 3] * 2 - assert list(wresources.all_rsets["gpu_share"]) == ([0] * 4 + [1] * 4) * 2 + assert list(wresources.all_rsets["gpu_id"]) == [0, 0, 1, 1, 2, 2, 3, 3] * 2 + assert list(wresources.all_rsets["gpu_share"]) == [0, 1, 0, 1, 0, 1, 0, 1] * 2 - wresources.set_rset_team([0, 1, 4, 5]) + wresources.set_rset_team([0, 1, 2, 3]) wresources.set_env_to_gpus("LIBE_TEST_GPUS_1") assert os.environ["LIBE_TEST_GPUS_1"] == "0,1" os.environ.pop("LIBE_TEST_GPUS_1") @@ -938,7 +938,7 @@ def test_wresources_set_gpus_with_sharing(): def test_gpu_sharing_executor_env_mapping(): wresources = _setup_wresources_gpus(4, gpu_sharing_factor=2, num_resource_sets=16) - wresources.set_rset_team([4]) + wresources.set_rset_team([1]) class Task: def _add_to_env(self, name, value): @@ -952,9 +952,9 @@ def _add_to_env(self, name, value): def test_gpu_sharing_mapping_with_gpu_groups(): wresources = _setup_wresources_gpus(4, gpu_sharing_factor=2, num_resource_sets=16, gpus_per_group=2) - assert list(wresources.all_rsets["slot"]) == [0, 1] * 8 - assert list(wresources.all_rsets["gpu_id"]) == [0, 1, 2, 3, 0, 1, 2, 3] * 2 - assert list(wresources.all_rsets["gpu_share"]) == ([0] * 4 + [1] * 4) * 2 + assert list(wresources.all_rsets["slot"]) == [0, 1, 2, 3] * 4 + assert list(wresources.all_rsets["gpu_id"]) == [0, 0, 1, 1, 2, 2, 3, 3] * 2 + assert list(wresources.all_rsets["gpu_share"]) == [0, 1, 0, 1, 0, 1, 0, 1] * 2 def test_wresources_set_gpus_x2(): diff --git a/libensemble/tools/alloc_support.py b/libensemble/tools/alloc_support.py index 6fda1e6b2..40b7eaf49 100644 --- a/libensemble/tools/alloc_support.py +++ b/libensemble/tools/alloc_support.py @@ -1,4 +1,5 @@ import logging +import math import numpy as np @@ -145,7 +146,7 @@ def _req_resources_sim(self, libE_info, user_params, H, H_rows): """Determine required resources for a sim work unit""" use_gpus = None if "resource_sets" in H.dtype.names: - num_rsets_req = np.max(H[H_rows]["resource_sets"]) # sim rsets + num_rsets_req = int(np.max(H[H_rows]["resource_sets"])) # sim rsets elif "num_procs" in H.dtype.names and self.resources: procs_per_rset = self.resources.resource_manager.procs_per_rset num_rsets_req = AllocSupport._convert_rows_to_rsets( @@ -155,20 +156,36 @@ def _req_resources_sim(self, libE_info, user_params, H, H_rows): num_rsets_req = 1 if "num_gpus" in H.dtype.names: gpus_per_rset = self.resources.resource_manager.gpus_per_rset - num_rsets_req_for_gpus = AllocSupport._convert_rows_to_rsets( - libE_info, user_params, H, H_rows, gpus_per_rset, "num_gpus" - ) - if num_rsets_req_for_gpus > 0: - use_gpus = True + gpu_sharing_factor = getattr(self.resources.resource_manager, "gpu_sharing_factor", 1) + if gpu_sharing_factor > 1 and self.resources: + max_gpus = float(np.max(H[H_rows]["num_gpus"])) + if max_gpus > 0: + use_gpus = True + shares_needed = ( + math.ceil(max_gpus * gpu_sharing_factor / gpus_per_rset) + if gpus_per_rset + else math.ceil(max_gpus * gpu_sharing_factor) + ) + user_params.append(shares_needed) + libE_info["num_gpus"] = max_gpus + num_rsets_req_for_gpus = shares_needed + else: + libE_info["num_gpus"] = max_gpus + num_rsets_req_for_gpus = 0 + else: + num_rsets_req_for_gpus = AllocSupport._convert_rows_to_rsets( + libE_info, user_params, H, H_rows, gpus_per_rset, "num_gpus" + ) + if num_rsets_req_for_gpus > 0: + use_gpus = True num_rsets_req = max(num_rsets_req, num_rsets_req_for_gpus) return num_rsets_req, use_gpus def _req_resources_gen(self, libE_info, user_params): """Determine required resources for a gen work unit""" - # We could also have libE_specs defaults (gen_num_procs, gen_num_gpus) - passed by libE_info use_gpus = None num_rsets_req = self.persis_info.get("gen_resources", 0) - use_gpus = self.persis_info.get("gen_use_gpus", None) # can be overwritten below + use_gpus = self.persis_info.get("gen_use_gpus", None) if not num_rsets_req: gen_nprocs = self.persis_info.get("gen_num_procs", self.def_gen_num_procs) if gen_nprocs and self.resources: @@ -179,11 +196,28 @@ def _req_resources_gen(self, libE_info, user_params): gen_ngpus = self.persis_info.get("gen_num_gpus", self.def_gen_num_gpus) if gen_ngpus and self.resources: gpus_per_rset = self.resources.resource_manager.gpus_per_rset - num_rsets_req_for_gpus = AllocSupport._convert_to_rsets( - libE_info, user_params, gpus_per_rset, gen_ngpus, "num_gpus" - ) - if num_rsets_req_for_gpus > 0: - use_gpus = True + gpu_sharing_factor = getattr(self.resources.resource_manager, "gpu_sharing_factor", 1) + if gpu_sharing_factor > 1: + gpus_val = float(gen_ngpus) + if gpus_val > 0: + use_gpus = True + shares_needed = ( + math.ceil(gpus_val * gpu_sharing_factor / gpus_per_rset) + if gpus_per_rset + else math.ceil(gpus_val * gpu_sharing_factor) + ) + user_params.append(shares_needed) + libE_info["num_gpus"] = gpus_val + num_rsets_req_for_gpus = shares_needed + else: + libE_info["num_gpus"] = gpus_val + num_rsets_req_for_gpus = 0 + else: + num_rsets_req_for_gpus = AllocSupport._convert_to_rsets( + libE_info, user_params, gpus_per_rset, gen_ngpus, "num_gpus" + ) + if num_rsets_req_for_gpus > 0: + use_gpus = True num_rsets_req = max(num_rsets_req, num_rsets_req_for_gpus) return num_rsets_req, use_gpus @@ -393,7 +427,7 @@ def _check_H_fields(H_fields): @staticmethod def _convert_rows_to_rsets(libE_info, user_params, H, H_rows, units_per_rset, units_str): """Convert num_procs & num_gpus requirements to resource sets for sim functions""" - max_num_units = int(np.max(H[H_rows][units_str])) + max_num_units = float(np.max(H[H_rows][units_str])) num_rsets_req = AllocSupport._convert_to_rsets(libE_info, user_params, units_per_rset, max_num_units, units_str) return num_rsets_req @@ -403,7 +437,12 @@ def _convert_to_rsets(libE_info, user_params, units_per_rset, num_units, units_s user_params.append(num_units) if num_units > 0: try: - num_rsets_req = num_units // units_per_rset + (num_units % units_per_rset > 0) + if isinstance(num_units, float) and not num_units.is_integer(): + num_rsets_req = math.ceil(num_units / units_per_rset) + else: + num_rsets_req = math.ceil(float(num_units) / units_per_rset) if units_per_rset else 0 + if float(num_units).is_integer() and isinstance(units_per_rset, int): + num_rsets_req = int(num_units) // units_per_rset + (int(num_units) % units_per_rset > 0) except ZeroDivisionError: raise InsufficientResourcesError( f"There are zero {units_str} per resource set (worker). Use fewer workers or more resources" From af84bcf795fc28216f7467b08cde5c7ec2a3a804 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 27 Oct 2026 15:01:11 -0500 Subject: [PATCH 3/8] mpi_runner also needs to consider gpus_per_rset_per_node now, when calculating available GPUs --- libensemble/executors/mpi_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/executors/mpi_runner.py b/libensemble/executors/mpi_runner.py index c79b6aee3..7d8fbb4fb 100644 --- a/libensemble/executors/mpi_runner.py +++ b/libensemble/executors/mpi_runner.py @@ -174,7 +174,7 @@ def _assign_gpus(self, task, resources, nprocs, nnodes, ppn, ngpus, extra_args, # gpus per node for this worker. if wresources.doihave_gpus(): - gpus_avail_per_node = len(set(wresources.gpu_ids_on_node)) + gpus_avail_per_node = len(set(wresources.gpu_ids_on_node)) * wresources.gpus_per_rset_per_node else: gpus_avail_per_node = 0 @@ -251,7 +251,7 @@ def _get_min_nodes(self, nprocs, ppn, nnodes, ngpus, resources): if nprocs: proc_min_nodes = (nprocs + procs_on_node - 1) // procs_on_node if ngpus: - gpus_on_node = len(set(wresources.gpu_ids_on_node)) + gpus_on_node = len(set(wresources.gpu_ids_on_node)) * wresources.gpus_per_rset_per_node import math try: From f60743396758d02978057bb7ebd71d0b84513c4c Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 27 Oct 2026 17:48:50 -0500 Subject: [PATCH 4/8] check_gpu_setting in test_support needs it's computation for requested gpus adjusted --- libensemble/tools/test_support.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libensemble/tools/test_support.py b/libensemble/tools/test_support.py index 4a5b41657..2f22871e3 100644 --- a/libensemble/tools/test_support.py +++ b/libensemble/tools/test_support.py @@ -204,7 +204,10 @@ def check_gpu_setting(task, assert_setting=True, print_setting=False, resources= # Get expected numbers if cmd_line: - expected_nums = _safe_min(len(set(wresources.gpu_ids_on_node)), wresources.gen_ngpus) + expected_nums = _safe_min( + len(set(wresources.gpu_ids_on_node)) * wresources.gpus_per_rset_per_node, + wresources.gen_ngpus, + ) if gpus_per_task: stype = "runline option: gpus per task" expected_nums //= int(ppn) From 80bc6c8ddb878aad30d444a114380d4c27b49fb8 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 27 Oct 2026 18:09:47 -0500 Subject: [PATCH 5/8] normalize to integer when appropriate --- libensemble/executors/mpi_runner.py | 4 ++++ libensemble/tools/test_support.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/libensemble/executors/mpi_runner.py b/libensemble/executors/mpi_runner.py index 7d8fbb4fb..29b47b2da 100644 --- a/libensemble/executors/mpi_runner.py +++ b/libensemble/executors/mpi_runner.py @@ -133,6 +133,8 @@ def _local_runner_set_gpus(self, task, wresources, extra_args, gpus_per_node, pp arg_type = self.default_gpu_arg_type if arg_type is not None: gpu_value = gpus_per_node // ppn if arg_type == "option_gpus_per_task" else gpus_per_node + if isinstance(gpu_value, float) and gpu_value.is_integer(): + gpu_value = int(gpu_value) gpu_setting_name = self.default_gpu_args[arg_type] jassert(gpu_setting_name is not None, f"No default gpu_setting_name for {arg_type}") extra_args = self._set_gpu_cli_option(wresources, extra_args, gpu_setting_name, gpu_value) @@ -223,6 +225,8 @@ def _assign_gpus(self, task, resources, nprocs, nnodes, ppn, ngpus, extra_args, elif gpu_setting_type in ["option_gpus_per_node", "option_gpus_per_task"]: gpu_value = gpus_per_node // ppn if gpu_setting_type == "option_gpus_per_task" else gpus_per_node + if isinstance(gpu_value, float) and gpu_value.is_integer(): + gpu_value = int(gpu_value) gpu_setting_name = self.platform_info.get("gpu_setting_name", self._get_default_arg(gpu_setting_type)) extra_args = self._set_gpu_cli_option(wresources, extra_args, gpu_setting_name, gpu_value) diff --git a/libensemble/tools/test_support.py b/libensemble/tools/test_support.py index 2f22871e3..03ce266bf 100644 --- a/libensemble/tools/test_support.py +++ b/libensemble/tools/test_support.py @@ -211,8 +211,12 @@ def check_gpu_setting(task, assert_setting=True, print_setting=False, resources= if gpus_per_task: stype = "runline option: gpus per task" expected_nums //= int(ppn) + if isinstance(expected_nums, float) and expected_nums.is_integer(): + expected_nums = int(expected_nums) else: stype = "runline option: gpus per node" + if isinstance(expected_nums, float) and expected_nums.is_integer(): + expected_nums = int(expected_nums) expected_nums = expected_nums if _set_gpus(task, wresources) else None if expected_nums is not None: expected = _get_expected_output(expected_setting, expected_nums) From 01b6f7e4aad723074571d4d327eabb80b03027a9 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 28 Oct 2026 13:40:31 -0500 Subject: [PATCH 6/8] regression test for sharing GPUs, or a task requesting half a GPU to two GPUs. unit tests for gpu_sharing_factor, gpus_per_group, building gpu bins, capacity handling for gpus, "pack" and "spread" policies, and fractional-gpu support in alloc_support --- .../test_GPU_sharing_variable_resources.py | 123 +++++++++ .../unit_tests/test_resource_scheduler.py | 235 ++++++++++++++++++ .../tests/unit_tests/test_resources.py | 93 +++++++ 3 files changed, 451 insertions(+) create mode 100644 libensemble/tests/regression_tests/test_GPU_sharing_variable_resources.py diff --git a/libensemble/tests/regression_tests/test_GPU_sharing_variable_resources.py b/libensemble/tests/regression_tests/test_GPU_sharing_variable_resources.py new file mode 100644 index 000000000..eaf3fe0ff --- /dev/null +++ b/libensemble/tests/regression_tests/test_GPU_sharing_variable_resources.py @@ -0,0 +1,123 @@ +""" +Tests GPU sharing with physical-exclusive scheduling and pack/spread policies. + +This regression test mimics a user workflow: a persistent generator produces +simulations that request fractional GPUs (0.5, 1.0, 1.5, 2.0) which, with +``gpu_sharing_factor=2``, map to 1-4 logical GPU shares. The ensemble is +configured with 2 mocked nodes, 4 GPUs per node, and ``scheduler_opts`` to +exercise pack/spread packing. The simulator (``gpu_variable_resources_from_gen``) +validates GPU assignment via ``check_gpu_setting``. + +Execute via (e.g. 9 workers): + mpiexec -np 10 python test_GPU_sharing_variable_resources.py + python test_GPU_sharing_variable_resources.py --nworkers 9 +""" + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: mpi local +# TESTSUITE_NPROCS: 10 + +import numpy as np + +from libensemble import Ensemble +from libensemble.executors.mpi_executor import MPIExecutor +from libensemble.message_numbers import EVAL_GEN_TAG, FINISHED_PERSISTENT_GEN_TAG, PERSIS_STOP, STOP_TAG +from libensemble.sim_funcs import six_hump_camel +from libensemble.sim_funcs.var_resources import gpu_variable_resources_from_gen as sim_f +from libensemble.specs import GenSpecs, LibeSpecs, SimSpecs +from libensemble.tools.persistent_support import PersistentSupport + + +def uniform_sample_sharing(_, persis_info, gen_specs, libE_info): + """Persistent generator that cycles through fractional GPU requests. + + Each batch emits ``num_gpus`` values from the repeating cycle + ``[0.5, 1.0, 1.5, 2.0]`` which, with ``gpu_sharing_factor=2``, map to + ``[1, 2, 3, 4]`` logical shares. ``0.5`` exercises the fractional + (partial-GPU) path, ``1.0`` a whole single GPU, ``1.5`` whole+remainder, + and ``2.0`` an exclusive two-GPU request. ``num_procs`` is fixed at 1 so + GPU-driven resource sizing dominates. The generator also draws ``x`` + uniformly from ``gen_specs["user"]["lb"/"ub"]`` so the sim can run + ``six_hump_camel``. Used by the ensemble integration portion of the + test to drive all share-aware branches in ``ResourceScheduler`` and + ``AllocSupport``. + """ + # Extract domain and batch size from gen_specs + ub = gen_specs["user"]["ub"] + lb = gen_specs["user"]["lb"] + n = len(lb) + b = gen_specs["initial_batch_size"] + from libensemble.tools import get_rng + + # Set up RNG and persistent support channel to manager + rng = get_rng(gen_specs, libE_info) + ps = PersistentSupport(libE_info, EVAL_GEN_TAG) + tag = None + # Cycle covers all interesting share counts: 1,2,3,4 shares + cycle = [0.5, 1.0, 1.5, 2.0] + idx = 0 + while tag not in [STOP_TAG, PERSIS_STOP]: + # Build output array for this batch + H_o = np.zeros(b, dtype=gen_specs["out"]) + H_o["x"] = rng.uniform(lb, ub, (b, n)) + # Round-robin through fractional cycle so every batch mixes types + ngpus = [cycle[(idx + i) % len(cycle)] for i in range(b)] + idx += b + H_o["num_gpus"] = ngpus + H_o["num_procs"] = np.ones(b, dtype=int) # keep CPU sizing trivial + print(f"GEN created {b} sims requiring {ngpus} GPUs (shares {[int(v * 2) for v in ngpus]})", flush=True) + tag, Work, calc_in = ps.send_recv(H_o) + if hasattr(calc_in, "__len__"): + b = len(calc_in) + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG + + +if __name__ == "__main__": + # Register dummy app for MPIExecutor (six_hump_camel is Python, no GPU needed) + six_hump_camel_app = six_hump_camel.__file__ + exctr = MPIExecutor() + exctr.register_app(full_path=six_hump_camel_app, app_name="six_hump_camel") + + # Ensemble setup: parse --nworkers/--comms, ensure enough rsets for fractional tasks + gpu_test = Ensemble(parse_args=True, executor=exctr) + nworkers = gpu_test.nworkers + nsets = max(gpu_test.nworkers - 1, 8) # need >=8 shares to host max 4-share task + gpu_test.libE_specs = LibeSpecs( + num_resource_sets=nsets, + resource_info={"cores_on_node": (32, 64), "gpus_on_node": 4}, + gpu_sharing_factor=2, + scheduler_opts={"gpu_packing": "pack"}, + sim_dirs_make=True, + reuse_output_dir=True, + ensemble_dir_path="./ensemble_GPU_sharing_variable_w" + str(nworkers) + str(gpu_test.libE_specs.comms), + ) + + # Simulator: auto-assign GPUs, validates via check_gpu_setting in var_resources + gpu_test.sim_specs = SimSpecs( + sim_f=sim_f, + inputs=["x"], + out=[("f", float)], + user={"dry_run": False}, # set True to only log runlines without executing + ) + # Generator: persistent, cycles fractional num_gpus (float) to drive sharing logic + gpu_test.gen_specs = GenSpecs( + gen_f=uniform_sample_sharing, + persis_in=["f", "x", "sim_id"], + out=[("num_procs", int), ("num_gpus", float), ("x", float, 2)], + initial_batch_size=nworkers - 1, # fill all sim workers initially + batch_evaluate_same_priority=False, + async_return=False, + user={ + "lb": np.array([-3, -2]), + "ub": np.array([3, 2]), + }, + ) + + if gpu_test.ready(): + gpu_test.run(sim_max=12, wallclock_max=300) # 12 sims covers 3 full cycles + + # Manager post-checks: flag 0 and save history + if gpu_test.is_manager: + assert gpu_test.flag == 0 # ensemble completed without manager exception + gpu_test.save_output(__file__) + print("GPU sharing regression test passed", flush=True) diff --git a/libensemble/tests/unit_tests/test_resource_scheduler.py b/libensemble/tests/unit_tests/test_resource_scheduler.py index 762f07ead..c28e0c3a9 100644 --- a/libensemble/tests/unit_tests/test_resource_scheduler.py +++ b/libensemble/tests/unit_tests/test_resource_scheduler.py @@ -1,9 +1,11 @@ +import os import sys import time import numpy as np import pytest +from libensemble.resources.resources import Resources from libensemble.resources.scheduler import InsufficientFreeResources, InsufficientResourcesError, ResourceScheduler @@ -661,6 +663,235 @@ def test_schedule_find_gaps_2nodes_withgpus(): del resources +def _setup_resources_sharing(gpu_sharing_factor=2, gpus_per_group=None, sched_opts=None, num_resource_sets=16): + """Create a mocked 2-node, 4-GPU/node resource set for helper tests. + + Mocks SLURM nodelist ``knl-[0020,0036]`` (2 nodes) with + ``gpus_on_node=4`` and ``cores_on_node=(32,64)``-equivalent layout via + ``ResourceInfo``. With ``gpu_sharing_factor=2`` each physical GPU is + split into 2 logical shares, yielding ``8`` GPU shares per node and + ``16`` total for the default ``num_resource_sets=16`` (``8`` per node). + Returns the ``ResourceManager`` and a ``ResourceScheduler`` bound to it. + + Args: + gpu_sharing_factor: Logical shares per physical GPU (default 2). + gpus_per_group: Optional cap on GPUs per scheduler group, scaled by + ``gpu_sharing_factor`` in ``RSetResources.get_group_list``. + sched_opts: Dict of scheduler options, notably ``gpu_packing`` + ``\"pack\"`` (co-locate fractional shares) or ``\"spread\"``. + num_resource_sets: Total rsets to create; ``rsets_per_node`` is + derived as ``ceil(num_resource_sets/num_nodes)``. + + Returns: + Tuple of (resource_manager, scheduler) ready for direct calls to + ``_build_gpu_bins`` etc. Caller is responsible for cleaning the + mocked env var if needed. + """ + # Mock a 2-node SLURM nodelist; no real hardware needed + os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "knl-[0020,0036]" + resource_info = {"nodelist_env_slurm": "LIBE_RESOURCES_TEST_NODE_LIST", "gpus_on_node": 4} + libE_specs = { + "resource_info": resource_info, + "dedicated_mode": False, # keep libEnsemble ranks out of the nodelist + "gpu_sharing_factor": gpu_sharing_factor, + "gpus_per_group": gpus_per_group, + "num_resource_sets": num_resource_sets, + } + # Instantiate Resources and its ResourceManager (holds RSetResources) + res = Resources(libE_specs=libE_specs, top_level_dir=os.getcwd()) + res.set_resource_manager(num_workers=8) + # Bind scheduler to the mocked manager + sched = ResourceScheduler(user_resources=res.resource_manager, sched_opts=sched_opts or {}) + return res.resource_manager, sched + + +def test_build_gpu_bins(): + """Unit coverage for ``ResourceScheduler._build_gpu_bins``. + + Verifies the per-group GPU bin construction on a fresh 2-node, 16-rset + layout (sf=2). Each physical GPU should own exactly ``sf`` rsets, so + group 1 is expected to contain bins ``{0:[0,1], 1:[2,3], 2:[4,5], + 3:[6,7]}`` and group 2 similarly ``11:[10,11]`` … ``14,15``. This + structure is the foundation for all physical-exclusive / packing + decisions. + """ + # Fresh mocked resources: 2 groups, 8 GPU shares each + _, sched = _setup_resources_sharing() + # Get GPU-only availability by group + valid = sched.get_avail_rsets_by_group() + valid = sched.filter_for_rset_type(valid, use_gpus=True) + # Build bins: group -> gpu_id -> [rset_ids] + bins = sched._build_gpu_bins(valid) + # Two groups (nodes) expected + assert set(bins.keys()) == {1, 2} + # Group 1: 4 physical GPUs each with 2 shares + assert bins[1][0] == [0, 1] # GPU 0 shares + assert bins[1][1] == [2, 3] # GPU 1 shares + assert bins[1][2] == [4, 5] # GPU 2 shares + assert bins[1][3] == [6, 7] # GPU 3 shares + # Spot-check group 2 last GPU + assert bins[2][3] == [14, 15] + + +def test_can_group_satisfy_gpu(): + """Unit coverage for ``ResourceScheduler._can_group_satisfy_gpu``. + + Exercises every branch of the satisfiability check: + + * **Fresh node:** ``need`` 1,2,4,8 (1 share … all 8 shares on one node) + should succeed, 9 should fail. + * **After partial allocation:** consume 2 shares (one whole GPU), then + 6 shares still fits but 7 does not. + * **After one-share allocation:** 1 share still fits, 2 still needs a + fully-free GPU. + * **Synthetic bins:** a partially-occupied GPU ``{0:[0]}`` with others + fully free still satisfies 1 and 3 shares via the ``leftover_fully`` + and ``partial >= rem`` paths. + * **Fragmented failure:** bins each with a single free share cannot + satisfy a whole GPU (need 2) or 5 shares, hitting the ``total_free`` + fallback. + """ + # Start from a fully-free 8-share-per-node layout + _, sched = _setup_resources_sharing() + valid = sched.filter_for_rset_type(sched.get_avail_rsets_by_group(), use_gpus=True) + bins = sched._build_gpu_bins(valid) + # Fresh node satisfiability: 1 share up to full node, 9 exceeds capacity + assert sched._can_group_satisfy_gpu(1, 1, bins) is True # single share + assert sched._can_group_satisfy_gpu(1, 2, bins) is True # one whole GPU (sf=2) + assert sched._can_group_satisfy_gpu(1, 4, bins) is True # two whole GPUs + assert sched._can_group_satisfy_gpu(1, 8, bins) is True # entire node + assert sched._can_group_satisfy_gpu(1, 9, bins) is False # over capacity + # Allocate one whole GPU (2 shares) and re-check remaining capacity + sched.assign_resources(rsets_req=2, use_gpus=True) + valid2 = sched.filter_for_rset_type(sched.get_avail_rsets_by_group(), use_gpus=True) + bins2 = sched._build_gpu_bins(valid2) + assert sched._can_group_satisfy_gpu(1, 6, bins2) is True # 6 of remaining 6 fits + assert sched._can_group_satisfy_gpu(1, 7, bins2) is False # 7 needs more than free + # One share taken from fresh manager: still room for 1 or 2 on same node + sched2 = ResourceScheduler(user_resources=_setup_resources_sharing()[0], sched_opts={"gpu_packing": "pack"}) + sched2.assign_resources(rsets_req=1, use_gpus=True) + valid3 = sched2.filter_for_rset_type(sched2.get_avail_rsets_by_group(), use_gpus=True) + bins3 = sched2._build_gpu_bins(valid3) + assert sched2._can_group_satisfy_gpu(1, 1, bins3) is True + assert sched2._can_group_satisfy_gpu(1, 2, bins3) is True + # Synthetic bins with one partially-occupied GPU: triggers leftover_fully branch + bins_partial = {1: {0: [0], 1: [2, 3], 2: [4, 5], 3: [6, 7]}} + assert sched._can_group_satisfy_gpu(1, 1, bins_partial) is True # uses leftover fully-free GPU + assert sched._can_group_satisfy_gpu(1, 3, bins_partial) is True # needs 1 whole +1, partial suffices + # Highly fragmented: each GPU has only 1 free share, no whole GPU left + bins_frag = {1: {0: [0], 1: [2], 2: [4], 3: [6]}} + assert sched._can_group_satisfy_gpu(1, 2, bins_frag) is False # need whole GPU + assert sched._can_group_satisfy_gpu(1, 5, bins_frag) is False # total_free insufficient + + +def test_select_rsets_for_gpu_group_pack_spread(): + """Unit coverage for ``ResourceScheduler._select_rsets_for_gpu_group``. + + Checks both ``pack`` and ``spread`` policies inside a single group: + + * **Fresh group:** 1 share picks ``[0]``; 2 shares (one whole GPU) + picks two rsets on the same ``gpu_id``. + * **Pack after one taken:** with ``[0]`` consumed, pack co-locates the + next 1-share request onto the same GPU ``[1]`` to preserve fully-free + GPUs. + * **Spread after one taken:** spread instead picks the least-occupied + GPU ``[2]`` (next fully-free GPU). + * **Whole + remainder:** 4 shares picks 2 whole GPUs (distinct ids), 3 + shares picks whole+1 (also 2 distinct ids). These validate the + ``whole``/``rem`` split and the candidate sorting. + """ + # Pack policy fresh group + _, sched_pack = _setup_resources_sharing(sched_opts={"gpu_packing": "pack"}) + valid = sched_pack.filter_for_rset_type(sched_pack.get_avail_rsets_by_group(), use_gpus=True) + bins = sched_pack._build_gpu_bins(valid) + # 1 share on fresh node: picks first rset of first GPU (lowest gid) + sel = sched_pack._select_rsets_for_gpu_group(1, 1, bins) + assert sel == [0] + # 2 shares = one whole GPU: both rsets on same physical GPU + sel2 = sched_pack._select_rsets_for_gpu_group(1, 2, bins) + assert len(sel2) == 2 + assert sched_pack.resources.rsets[sel2[0]]["gpu_id"] == sched_pack.resources.rsets[sel2[1]]["gpu_id"] + # Pack co-location: after [0] taken, next 1-share should fill same GPU [1] + sched_pack.assign_resources(rsets_req=1, use_gpus=True) + valid2 = sched_pack.filter_for_rset_type(sched_pack.get_avail_rsets_by_group(), use_gpus=True) + bins2 = sched_pack._build_gpu_bins(valid2) + sel_pack = sched_pack._select_rsets_for_gpu_group(1, 1, bins2) + assert sel_pack == [1] # preserves fully-free GPUs + # Spread: same scenario should pick next fully-free GPU instead + _, sched_spread = _setup_resources_sharing(sched_opts={"gpu_packing": "spread"}) + sched_spread.assign_resources(rsets_req=1, use_gpus=True) + valid_s = sched_spread.filter_for_rset_type(sched_spread.get_avail_rsets_by_group(), use_gpus=True) + bins_s = sched_spread._build_gpu_bins(valid_s) + sel_spread = sched_spread._select_rsets_for_gpu_group(1, 1, bins_s) + assert sel_spread == [2] # least-occupied fully-free GPU + # Whole GPUs: 4 shares => 2 distinct physical GPUs + sel_whole = sched_pack._select_rsets_for_gpu_group(1, 4, bins) + assert len(sel_whole) == 4 + assert len(set(sched_pack.resources.rsets[sel_whole]["gpu_id"])) == 2 + # Whole+remainder: 3 shares => 1 whole GPU +1 share on another GPU + sel_rem = sched_pack._select_rsets_for_gpu_group(1, 3, bins) + assert len(sel_rem) == 3 + assert len(set(sched_pack.resources.rsets[sel_rem]["gpu_id"])) == 2 + + +def test_assign_gpu_sharing_resources(): + """Unit + integration coverage for ``_assign_gpu_sharing_resources``. + + Validates the end-to-end physical-exclusive allocator: + + * **Pack co-location:** two consecutive 1-share ``assign_resources`` + calls land on the same ``gpu_id`` and same ``group`` (node). + * **Spread node selection:** same two calls with ``gpu_packing=spread`` + land on different groups (nodes) because ``feasible`` groups are + sorted by ``-total_free``. + * **Exclusive multi-GPU:** 4 shares → 2 distinct physical GPUs on one + node; 3 shares → 2 distinct GPUs (whole+rem). + * **Error paths:** requesting 20 shares exceeds total GPU rsets and + raises ``InsufficientResourcesError``/``InsufficientFreeResources``; + invalid ``gpu_packing=\"bad\"`` raises ``ValueError``. + * **GPU groups:** with ``gpus_per_group=2`` (scaled by sharing factor) + an 8-share request correctly spans the expanded group threshold. + """ + # Pack co-location: two 1-share tasks should stack on same physical GPU/node + gres, sched = _setup_resources_sharing(sched_opts={"gpu_packing": "pack"}) + team = sched.assign_resources(rsets_req=1, use_gpus=True) + assert team == [0] # first share on GPU 0, group 1 + assert sched.gpu_rsets_free == 15 # one GPU share consumed + team2 = sched.assign_resources(rsets_req=1, use_gpus=True) + assert team2 == [1] # second share packs onto same GPU + assert sched.resources.rsets[team[0]]["gpu_id"] == sched.resources.rsets[team2[0]]["gpu_id"] + assert sched.resources.rsets[team[0]]["group"] == sched.resources.rsets[team2[0]]["group"] + # Spread node selection: same 1-share pair should go to different nodes + _, sched_spread = _setup_resources_sharing(sched_opts={"gpu_packing": "spread"}) + t1 = sched_spread.assign_resources(rsets_req=1, use_gpus=True) + t2 = sched_spread.assign_resources(rsets_req=1, use_gpus=True) + assert sched_spread.resources.rsets[t1[0]]["group"] != sched_spread.resources.rsets[t2[0]]["group"] + # Exclusive multi-GPU: 4 shares = 2 whole GPUs, must be distinct physical ids on one node + _, sched_exc = _setup_resources_sharing() # fresh pack scheduler + team_exc = sched_exc.assign_resources(rsets_req=4, use_gpus=True) + assert len(team_exc) == 4 + assert len(set(sched_exc.resources.rsets[team_exc]["gpu_id"])) == 2 + # Whole+remainder: 3 shares = 1 whole +1, also 2 distinct GPUs + team_frac = sched_exc.assign_resources(rsets_req=3, use_gpus=True) + assert len(team_frac) == 3 + assert len(set(sched_exc.resources.rsets[team_frac]["gpu_id"])) == 2 + # Error paths: over-request beyond total GPU shares (16) and invalid packing option + try: + sched_exc.assign_resources(rsets_req=20, use_gpus=True) + assert False, "should have raised Insufficient*" + except Exception as e: + assert e.__class__.__name__ in ("InsufficientResourcesError", "InsufficientFreeResources") + try: + _ = ResourceScheduler(user_resources=gres, sched_opts={"gpu_packing": "bad"}) + assert False, "invalid packing should raise ValueError" + except ValueError: + pass # expected: packing must be 'pack' or 'spread' + # GPU groups: gpus_per_group scaled by sf, 8 shares should fill one expanded group + _, sched_grp = _setup_resources_sharing(gpus_per_group=2) + team_grp = sched_grp.assign_resources(rsets_req=8, use_gpus=True) + assert len(team_grp) == 8 + + if __name__ == "__main__": test_request_zero_rsets() test_too_many_rsets() @@ -680,3 +911,7 @@ def test_schedule_find_gaps_2nodes_withgpus(): test_try1node_findon_2_or_4nodes() test_large_match_slots() test_schedule_find_gaps_2nodes_withgpus() + test_build_gpu_bins() + test_can_group_satisfy_gpu() + test_select_rsets_for_gpu_group_pack_spread() + test_assign_gpu_sharing_resources() diff --git a/libensemble/tests/unit_tests/test_resources.py b/libensemble/tests/unit_tests/test_resources.py index fb88a60f3..e836bfe39 100644 --- a/libensemble/tests/unit_tests/test_resources.py +++ b/libensemble/tests/unit_tests/test_resources.py @@ -1,10 +1,13 @@ import os import socket +import numpy as np + from libensemble.executors.mpi_runner import MPICH_MPIRunner from libensemble.resources.env_resources import EnvResources from libensemble.resources.mpi_resources import create_machinefile from libensemble.resources.resources import GlobalResources, Resources, ResourcesException +from libensemble.resources.scheduler import ResourceScheduler from libensemble.resources.worker_resources import ResourceManager, WorkerResources @@ -1017,6 +1020,94 @@ def test_wresources_set_no_gpus(): assert "LIBE_TEST_GPUS_1" not in os.environ +def _setup_resources_sharing_for_alloc(gpu_sharing_factor=2, num_resource_sets=16): + """Create mocked Resources for AllocSupport fractional tests. + + Mocks a 2-node SLURM nodelist with 4 GPUs/node and sf=2, yielding + 8 GPU shares per node (16 total). Returns the ResourceManager that + AllocSupport will use to convert fractional ``num_gpus`` to shares via + ``ceil(num_gpus*sf/gpus_per_rset)``. + """ + # Mock a 2-node SLURM nodelist; no real hardware needed + os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "knl-[0020,0036]" + resource_info = {"nodelist_env_slurm": "LIBE_RESOURCES_TEST_NODE_LIST", "gpus_on_node": 4} + libE_specs = { + "resource_info": resource_info, + "dedicated_mode": False, # keep libEnsemble ranks out of the nodelist + "gpu_sharing_factor": gpu_sharing_factor, + "num_resource_sets": num_resource_sets, + } + # Instantiate Resources and its ResourceManager (holds RSetResources) + res = Resources(libE_specs=libE_specs, top_level_dir=os.getcwd()) + res.set_resource_manager(num_workers=8) + return res.resource_manager + + +def test_alloc_support_fractional(): + """Unit coverage for ``AllocSupport`` fractional ``num_gpus`` handling. + + When ``gpu_sharing_factor>1`` the manager converts a fractional + ``num_gpus`` (physical GPUs) to logical shares via + ``ceil(num_gpus * sf / gpus_per_rset)``. This test drives: + + * ``_req_resources_sim`` with History rows ``0.5→1 share``, ``1.5→3``, + ``2.0→4`` and asserts ``use_gpus`` and ``libE_info[\"num_gpus\"]``. + * ``_req_resources_gen`` with ``persis_info[\"gen_num_gpus\"]`` ``0.5`` + and ``1.5`` (the ``def_gen_num_gpus`` path) likewise mapping to 1 and + 3 shares. + + Uses a lightweight ``AllocSupport`` instantiation with a mocked + ``Resources`` object to avoid a full ensemble run. + """ + # Use a mocked 16-rset manager with sf=2 (gpus_per_rset=1) + res_mgr = _setup_resources_sharing_for_alloc() + from libensemble.tools.alloc_support import AllocSupport + + # Minimal worker array needed by AllocSupport + W = np.zeros(3, dtype=[("persis_state", int), ("active", int)]) + libE_info = {} + user_params: list = [] + # History with fractional num_gpus: 0.5 and 1.5 physical GPUs + H = np.zeros(2, dtype=[("num_gpus", float), ("num_procs", int)]) + H["num_gpus"] = [0.5, 1.5] + H["num_procs"] = [1, 1] + support = AllocSupport(W, manage_resources=True, libE_info=libE_info, user_resources=Resources.resources) + # Inject mocked Resources with resource_manager + support.resources = type("obj", (), {"resource_manager": res_mgr, "resources": True})() + support.sched = ResourceScheduler(user_resources=res_mgr, sched_opts={"gpu_packing": "pack"}) + # 0.5 GPUs * sf2 / gpus_per_rset1 => ceil(1) = 1 share + nsets, use_gpus = support._req_resources_sim(libE_info, user_params, H, [0]) + assert nsets == 1 and use_gpus is True and libE_info["num_gpus"] == 0.5 + user_params = [] + libE_info = {} + # 1.5 GPUs => ceil(3) = 3 shares (1 whole GPU +1 share) + nsets2, _ = support._req_resources_sim(libE_info, user_params, H, [1]) + assert nsets2 == 3 + user_params = [] + libE_info = {} + # 2.0 GPUs => ceil(4) = 4 shares (2 whole GPUs) + H2 = np.zeros(1, dtype=[("num_gpus", float)]) + H2["num_gpus"] = [2.0] + nsets3, _ = support._req_resources_sim(libE_info, user_params, H2, [0]) + assert nsets3 == 4 + user_params = [] + libE_info = {} + # Generator path via persis_info gen_num_gpus (0.5 => 1 share) + support2 = AllocSupport( + W, manage_resources=True, persis_info={"gen_num_gpus": 0.5}, libE_info={}, user_resources=support.resources + ) + support2.resources = support.resources + support2.sched = support.sched + nsets4, _ = support2._req_resources_gen(libE_info, user_params) + assert nsets4 == 1 + user_params = [] + libE_info = {} + # Generator with 1.5 GPUs => 3 shares + support2.persis_info = {"gen_num_gpus": 1.5} + nsets5, _ = support2._req_resources_gen(libE_info, user_params) + assert nsets5 == 3 + + if __name__ == "__main__": setup_standalone_run() @@ -1050,4 +1141,6 @@ def test_wresources_set_no_gpus(): test_wresources_set_limit_gpus() test_wresources_set_no_gpus() + test_alloc_support_fractional() + teardown_standalone_run() From 9444c880da38ad06f449f9a26c21685572876e73 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 28 Oct 2026 14:00:19 -0500 Subject: [PATCH 7/8] fix alloc_support fractional test to avoid global Resources.resources Use local Resources instance instead of Resources.resources (None when run via run_tests.py -u isolation). Return Resources not just manager, pass directly to AllocSupport, and override scheduler for pack policy. --- libensemble/tests/unit_tests/test_resources.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libensemble/tests/unit_tests/test_resources.py b/libensemble/tests/unit_tests/test_resources.py index e836bfe39..2bb732085 100644 --- a/libensemble/tests/unit_tests/test_resources.py +++ b/libensemble/tests/unit_tests/test_resources.py @@ -1024,7 +1024,7 @@ def _setup_resources_sharing_for_alloc(gpu_sharing_factor=2, num_resource_sets=1 """Create mocked Resources for AllocSupport fractional tests. Mocks a 2-node SLURM nodelist with 4 GPUs/node and sf=2, yielding - 8 GPU shares per node (16 total). Returns the ResourceManager that + 8 GPU shares per node (16 total). Returns the ``Resources`` object that AllocSupport will use to convert fractional ``num_gpus`` to shares via ``ceil(num_gpus*sf/gpus_per_rset)``. """ @@ -1040,7 +1040,7 @@ def _setup_resources_sharing_for_alloc(gpu_sharing_factor=2, num_resource_sets=1 # Instantiate Resources and its ResourceManager (holds RSetResources) res = Resources(libE_specs=libE_specs, top_level_dir=os.getcwd()) res.set_resource_manager(num_workers=8) - return res.resource_manager + return res def test_alloc_support_fractional(): @@ -1059,8 +1059,8 @@ def test_alloc_support_fractional(): Uses a lightweight ``AllocSupport`` instantiation with a mocked ``Resources`` object to avoid a full ensemble run. """ - # Use a mocked 16-rset manager with sf=2 (gpus_per_rset=1) - res_mgr = _setup_resources_sharing_for_alloc() + # Use a mocked 16-rset Resources with sf=2 (gpus_per_rset=1) + res = _setup_resources_sharing_for_alloc() from libensemble.tools.alloc_support import AllocSupport # Minimal worker array needed by AllocSupport @@ -1071,10 +1071,10 @@ def test_alloc_support_fractional(): H = np.zeros(2, dtype=[("num_gpus", float), ("num_procs", int)]) H["num_gpus"] = [0.5, 1.5] H["num_procs"] = [1, 1] - support = AllocSupport(W, manage_resources=True, libE_info=libE_info, user_resources=Resources.resources) - # Inject mocked Resources with resource_manager - support.resources = type("obj", (), {"resource_manager": res_mgr, "resources": True})() - support.sched = ResourceScheduler(user_resources=res_mgr, sched_opts={"gpu_packing": "pack"}) + # Pass the mocked Resources directly (avoid global Resources.resources=None issue) + support = AllocSupport(W, manage_resources=True, libE_info=libE_info, user_resources=res) + # Override scheduler to enforce pack policy for deterministic shares + support.sched = ResourceScheduler(user_resources=res.resource_manager, sched_opts={"gpu_packing": "pack"}) # 0.5 GPUs * sf2 / gpus_per_rset1 => ceil(1) = 1 share nsets, use_gpus = support._req_resources_sim(libE_info, user_params, H, [0]) assert nsets == 1 and use_gpus is True and libE_info["num_gpus"] == 0.5 From f5d73e0dc05c81a0d03f62de0c3f2d8d6a7245e8 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 28 Oct 2026 15:44:46 -0500 Subject: [PATCH 8/8] cover fragmented remainder branch in gpu sharing scheduler Add test_select_rsets_fragmented_remainder for sf=4 where only partial GPUs are free. No single GPU satisfies rem, so _select_rsets_for_gpu_group must fragment across multiple GPUs (pack vs spread ordering) and raise InsufficientFreeResources when fragments insufficient. Covers previously uncovered else block scheduler.py:256-269. --- .../unit_tests/test_resource_scheduler.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/libensemble/tests/unit_tests/test_resource_scheduler.py b/libensemble/tests/unit_tests/test_resource_scheduler.py index c28e0c3a9..1d3891891 100644 --- a/libensemble/tests/unit_tests/test_resource_scheduler.py +++ b/libensemble/tests/unit_tests/test_resource_scheduler.py @@ -834,6 +834,62 @@ def test_select_rsets_for_gpu_group_pack_spread(): assert len(set(sched_pack.resources.rsets[sel_rem]["gpu_id"])) == 2 +def test_select_rsets_fragmented_remainder(): + """Covers fragmented-remainder path in ``_select_rsets_for_gpu_group``. + + This is the ``else`` branch at ``scheduler.py:256-269`` which was + previously uncovered. It triggers when only *partial* GPUs are available + — no single physical GPU has enough free shares to satisfy ``rem`` — so + the remainder must be assembled from fragments across multiple GPUs. + + * **Only partial GPUs:** with ``sf=4``, ``need=3`` (``whole=0, rem=3``) and + bins of size ``1,2,1`` no candidate satisfies ``len>=rem``; the method + fragments ``rem`` across GPUs. ``pack`` picks smallest bins first, + ``spread`` picks largest first — both are verified. + * **Whole + fragmented rem:** ``need=7`` (``whole=1, rem=3``) consumes one + fully-free GPU (4 shares) then fragments the remaining 3 as above. + * **Error:** fragmented capacity insufficient (need 3 but only 2 free) + correctly raises ``InsufficientFreeResources``. + """ + # Use sf=4 so rem can be 3 and fragmentation is observable (sf=2 never fragments) + res4, _ = _setup_resources_sharing(gpu_sharing_factor=4, num_resource_sets=32) + sched_pack = ResourceScheduler(user_resources=res4, sched_opts={"gpu_packing": "pack"}) + sched_spread = ResourceScheduler(user_resources=res4, sched_opts={"gpu_packing": "spread"}) + + # Fragmented remainder only: need=3, whole=0, rem=3, max bin size 2 <3 → no candidates + bins_frag = {1: {0: [0], 1: [4, 5], 2: [8]}} # lens 1,2,1 + sel_pack = sched_pack._select_rsets_for_gpu_group(1, 3, bins_frag) + # pack sorts by (len, gid): (1,0)->[0], (1,2)->[8], (2,1)->[4] => [0,8,4] sorted + assert sel_pack == [0, 4, 8] + sel_spread = sched_spread._select_rsets_for_gpu_group(1, 3, bins_frag) + # spread sorts by (-len, gid): (2,1)->[4,5] (take 2), (1,0)->[0] (take 1) => [4,5,0] sorted + assert sel_spread == [0, 4, 5] + + # Whole + fragmented remainder: need=7 → whole=1 (4 shares), rem=3 fragmented + bins_whole_frag = {1: {0: [0, 1, 2, 3], 1: [4], 2: [8], 3: [12, 13]}} + sel_pack2 = sched_pack._select_rsets_for_gpu_group(1, 7, bins_whole_frag) + # whole picks gid 0 (fully free): [0,1,2,3], remainder fragments 1,1,2 → [4,8,12] (pack order) + assert sel_pack2 == [0, 1, 2, 3, 4, 8, 12] + sel_spread2 = sched_spread._select_rsets_for_gpu_group(1, 7, bins_whole_frag) + # spread picks largest remainder first: gid3 (2) then gid1 (1) + assert sel_spread2 == [0, 1, 2, 3, 4, 12, 13] + + # Error path: not enough fragmented capacity for rem + bins_insufficient = {1: {0: [0], 1: [4]}} # total 2 < rem 3 + try: + sched_pack._select_rsets_for_gpu_group(1, 3, bins_insufficient) + assert False, "should raise InsufficientFreeResources" + except InsufficientFreeResources: + pass + # Also whole+rem insufficient: whole consumes 4, only 1 left but need 3 + bins_whole_insuf = {1: {0: [0, 1, 2, 3], 1: [4]}} + try: + sched_pack._select_rsets_for_gpu_group(1, 7, bins_whole_insuf) + assert False, "should raise InsufficientFreeResources" + except InsufficientFreeResources: + pass + + def test_assign_gpu_sharing_resources(): """Unit + integration coverage for ``_assign_gpu_sharing_resources``.