From 61ab1dc5f8023de478c8803d3c94266b4bd0bfef Mon Sep 17 00:00:00 2001 From: AHMETHAKANBEZIR1 Date: Wed, 21 Oct 2026 16:24:13 +0300 Subject: [PATCH 1/2] fix: retain zero kernels during connection normalization Co-authored-by: OpenAI Codex --- CHANGELOG.md | 3 + bindsnet/network/topology.py | 30 ++++---- .../network/test_zero_kernel_normalization.py | 71 +++++++++++++++++++ 3 files changed, 92 insertions(+), 12 deletions(-) create mode 100644 test/network/test_zero_kernel_normalization.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a6708bf..49885147 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ see the [GitHub releases / tags](https://github.com/BindsNET/bindsnet/releases). `environment.cue_reward`, `environment.dot_simulator` and `analysis.dotTrace_plotter`, which were missing. +- Normalizing an all-zero kernel in 1D/2D/3D convolutional and locally connected + connections keeps it zero instead of introducing NaNs. + ### Changed - CI: one test workflow (`python-app.yml`: job `build` on Python 3.13 plus a 3.11/3.12 matrix, Poetry 2.4.3 with dependency caching, superseded runs cancelled); diff --git a/bindsnet/network/topology.py b/bindsnet/network/topology.py index d74d5124..bdb97567 100644 --- a/bindsnet/network/topology.py +++ b/bindsnet/network/topology.py @@ -771,14 +771,15 @@ def normalize(self) -> None: # language=rst """ Normalize weights along the first axis according to total weight per target - neuron. + neuron. All-zero kernels remain zero. """ if self.norm is not None: # get a view and modify in place w = self.w.view(self.w.shape[0] * self.w.shape[1], self.w.shape[2]) for fltr in range(w.shape[0]): - w[fltr] *= self.norm / w[fltr].sum(0) + total = w[fltr].sum(0).masked_fill(w[fltr].abs().sum(0) == 0, 1) + w[fltr] *= self.norm / total def reset_state_variables(self) -> None: # language=rst @@ -930,7 +931,7 @@ def normalize(self) -> None: # language=rst """ Normalize weights along the first axis according to total weight per target - neuron. + neuron. All-zero kernels remain zero. """ if self.norm is not None: # get a view and modify in place @@ -939,7 +940,8 @@ def normalize(self) -> None: ) for fltr in range(w.shape[0]): - w[fltr] *= self.norm / w[fltr].sum(0) + total = w[fltr].sum(0).masked_fill(w[fltr].abs().sum(0) == 0, 1) + w[fltr] *= self.norm / total def reset_state_variables(self) -> None: # language=rst @@ -1110,7 +1112,7 @@ def normalize(self) -> None: # language=rst """ Normalize weights along the first axis according to total weight per target - neuron. + neuron. All-zero kernels remain zero. """ if self.norm is not None: # get a view and modify in place @@ -1120,7 +1122,8 @@ def normalize(self) -> None: ) for fltr in range(w.shape[0]): - w[fltr] *= self.norm / w[fltr].sum(0) + total = w[fltr].sum(0).masked_fill(w[fltr].abs().sum(0) == 0, 1) + w[fltr] *= self.norm / total def reset_state_variables(self) -> None: # language=rst @@ -1710,7 +1713,7 @@ def update(self, **kwargs) -> None: def normalize(self) -> None: """ Normalize weights so each target neuron has sum of connection weights equal to - ``self.norm``. + ``self.norm``. All-zero kernels remain zero. """ if self.norm is not None: # get a view and modify in-place @@ -1718,7 +1721,8 @@ def normalize(self) -> None: w = self.w.view(self.w.shape[0] * self.w.shape[1], self.w.shape[2]) for fltr in range(w.shape[0]): - w[fltr, :] *= self.norm / w[fltr, :].sum(0) + total = w[fltr].sum(0).masked_fill(w[fltr].abs().sum(0) == 0, 1) + w[fltr] *= self.norm / total def reset_state_variables(self) -> None: """ @@ -1861,7 +1865,7 @@ def update(self, **kwargs) -> None: def normalize(self) -> None: """ Normalize weights so each target neuron has sum of connection weights equal to - ``self.norm``. + ``self.norm``. All-zero kernels remain zero. """ if self.norm is not None: # get a view and modify in-place @@ -1869,7 +1873,8 @@ def normalize(self) -> None: w = self.w.view(self.w.shape[0] * self.w.shape[1], self.w.shape[2]) for fltr in range(w.shape[0]): - w[fltr, :] *= self.norm / w[fltr, :].sum(0) + total = w[fltr].sum(0).masked_fill(w[fltr].abs().sum(0) == 0, 1) + w[fltr] *= self.norm / total def reset_state_variables(self) -> None: """ @@ -2015,7 +2020,7 @@ def update(self, **kwargs) -> None: def normalize(self) -> None: """ Normalize weights so each target neuron has sum of connection weights equal to - ``self.norm``. + ``self.norm``. All-zero kernels remain zero. """ if self.norm is not None: # get a view and modify in-place @@ -2023,7 +2028,8 @@ def normalize(self) -> None: w = self.w.view(self.w.shape[0] * self.w.shape[1], self.w.shape[2]) for fltr in range(w.shape[0]): - w[fltr, :] *= self.norm / w[fltr, :].sum(0) + total = w[fltr].sum(0).masked_fill(w[fltr].abs().sum(0) == 0, 1) + w[fltr] *= self.norm / total def reset_state_variables(self) -> None: """ diff --git a/test/network/test_zero_kernel_normalization.py b/test/network/test_zero_kernel_normalization.py new file mode 100644 index 00000000..88505dfa --- /dev/null +++ b/test/network/test_zero_kernel_normalization.py @@ -0,0 +1,71 @@ +import pytest +import torch + +from bindsnet.network.nodes import Input, LIFNodes +from bindsnet.network.topology import ( + Conv1dConnection, + Conv2dConnection, + Conv3dConnection, + LocalConnection1D, + LocalConnection2D, + LocalConnection3D, +) + +CONNECTIONS = [ + (Conv1dConnection, 1), + (Conv2dConnection, 2), + (Conv3dConnection, 3), + (LocalConnection1D, 1), + (LocalConnection2D, 2), + (LocalConnection3D, 3), +] + + +def _connection(connection_type, dimensions, dtype, norm): + return connection_type( + Input(shape=(1, *((4,) * dimensions))), + LIFNodes(shape=(2, *((3,) * dimensions))), + kernel_size=2, + stride=1, + n_filters=2, + norm=norm, + w_dtype=dtype, + ) + + +@pytest.mark.parametrize(("connection_type", "dimensions"), CONNECTIONS) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float64]) +@pytest.mark.parametrize("norm", [None, 0.0, 2.0]) +def test_zero_kernel_normalization(connection_type, dimensions, dtype, norm): + connection = _connection(connection_type, dimensions, dtype, norm) + kernel_elements = 2**dimensions + rows = connection.w.numel() // kernel_elements + ramp = torch.arange(1, kernel_elements + 1, dtype=dtype) + weights = ramp.repeat(rows, 1) + weights[0].zero_() + with torch.no_grad(): + connection.w.copy_(weights.reshape_as(connection.w)) + parameter = connection.w + expected = weights.clone() + if norm is not None: + expected *= norm / (kernel_elements * (kernel_elements + 1) / 2) + for _ in range(2): + connection.normalize() + assert connection.w is parameter + torch.testing.assert_close(connection.w.reshape(rows, -1), expected) + if dtype == torch.float32: + output = connection.compute(torch.ones((1, 1, *((4,) * dimensions)))) + assert torch.isfinite(output).all() + + +@pytest.mark.parametrize(("connection_type", "dimensions"), CONNECTIONS) +def test_nonzero_signed_cancellation_is_not_a_zero_kernel(connection_type, dimensions): + connection = _connection(connection_type, dimensions, torch.float32, 2.0) + with torch.no_grad(): + weights = connection.w.reshape(-1, 2**dimensions) + weights[0].zero_() + weights[0, 0], weights[0, 1] = 1.0, -1.0 + connection.normalize() + actual = connection.w.reshape(-1, 2**dimensions) + assert torch.isposinf(actual[0, 0]) + assert torch.isneginf(actual[0, 1]) From 6720b5134d31c59efdd1404cec0fa696adf8fd18 Mon Sep 17 00:00:00 2001 From: Hananel Hazan Date: Thu, 22 Oct 2026 21:05:12 -0400 Subject: [PATCH 2/2] fix: vectorise kernel normalize and leave zero-sum kernels unchanged Review follow-up to #814. One tensor operation replaces the per-kernel loop in Conv1d/2d/3d and LocalConnection1D/2D/3D, and LocalConnection gets the same guard. Kernels whose weights sum to zero keep their weights instead of becoming NaN or infinite. CPU results match the old loop bit for bit; CUDA float32 can differ by rounding for kernels of 81+ weights. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 ++- bindsnet/network/topology.py | 59 ++++++++------ test/network/test_perf_equivalence.py | 80 +++++++++++++++++++ .../network/test_zero_kernel_normalization.py | 31 ++++++- 4 files changed, 149 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49885147..92c20716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,10 +22,15 @@ see the [GitHub releases / tags](https://github.com/BindsNET/bindsnet/releases). `environment.cue_reward`, `environment.dot_simulator` and `analysis.dotTrace_plotter`, which were missing. -- Normalizing an all-zero kernel in 1D/2D/3D convolutional and locally connected - connections keeps it zero instead of introducing NaNs. +- Normalizing a kernel whose weights sum to zero (for example an all-zero kernel) + in `LocalConnection` and the 1D/2D/3D convolutional and locally connected + connections leaves it unchanged instead of introducing NaNs or infinities. ### Changed +- Convolutional and locally connected connections normalize all kernels in one + operation instead of a Python loop (6 to 45 times faster on CPU). Results are + identical on CPU; on CUDA, float32 kernels of 81 or more weights can differ by + float32 rounding (about 1e-7) because the sum is taken in a different order. - CI: one test workflow (`python-app.yml`: job `build` on Python 3.13 plus a 3.11/3.12 matrix, Poetry 2.4.3 with dependency caching, superseded runs cancelled); `pythonpackage.yml` removed (it ran on every push to every branch and its diff --git a/bindsnet/network/topology.py b/bindsnet/network/topology.py index bdb97567..aa43c5cc 100644 --- a/bindsnet/network/topology.py +++ b/bindsnet/network/topology.py @@ -771,15 +771,16 @@ def normalize(self) -> None: # language=rst """ Normalize weights along the first axis according to total weight per target - neuron. All-zero kernels remain zero. + neuron. Kernels whose weights sum to zero are left unchanged. """ if self.norm is not None: # get a view and modify in place w = self.w.view(self.w.shape[0] * self.w.shape[1], self.w.shape[2]) - for fltr in range(w.shape[0]): - total = w[fltr].sum(0).masked_fill(w[fltr].abs().sum(0) == 0, 1) - w[fltr] *= self.norm / total + w_sum = w.sum(1, keepdim=True) + scale = self.norm / w_sum + scale.masked_fill_(w_sum == 0, 1.0) + w *= scale def reset_state_variables(self) -> None: # language=rst @@ -931,7 +932,7 @@ def normalize(self) -> None: # language=rst """ Normalize weights along the first axis according to total weight per target - neuron. All-zero kernels remain zero. + neuron. Kernels whose weights sum to zero are left unchanged. """ if self.norm is not None: # get a view and modify in place @@ -939,9 +940,10 @@ def normalize(self) -> None: self.w.shape[0] * self.w.shape[1], self.w.shape[2] * self.w.shape[3] ) - for fltr in range(w.shape[0]): - total = w[fltr].sum(0).masked_fill(w[fltr].abs().sum(0) == 0, 1) - w[fltr] *= self.norm / total + w_sum = w.sum(1, keepdim=True) + scale = self.norm / w_sum + scale.masked_fill_(w_sum == 0, 1.0) + w *= scale def reset_state_variables(self) -> None: # language=rst @@ -1112,7 +1114,7 @@ def normalize(self) -> None: # language=rst """ Normalize weights along the first axis according to total weight per target - neuron. All-zero kernels remain zero. + neuron. Kernels whose weights sum to zero are left unchanged. """ if self.norm is not None: # get a view and modify in place @@ -1121,9 +1123,10 @@ def normalize(self) -> None: self.w.shape[2] * self.w.shape[3] * self.w.shape[4], ) - for fltr in range(w.shape[0]): - total = w[fltr].sum(0).masked_fill(w[fltr].abs().sum(0) == 0, 1) - w[fltr] *= self.norm / total + w_sum = w.sum(1, keepdim=True) + scale = self.norm / w_sum + scale.masked_fill_(w_sum == 0, 1.0) + w *= scale def reset_state_variables(self) -> None: # language=rst @@ -1582,7 +1585,10 @@ def normalize(self) -> None: """ if self.norm is not None: w = self.w.view(self.source.n, self.target.n) - w *= self.norm / self.w.sum(0).view(1, -1) + w_sum = w.sum(0, keepdim=True) + scale = self.norm / w_sum + scale.masked_fill_(w_sum == 0, 1.0) + w *= scale def reset_state_variables(self) -> None: # language=rst @@ -1713,16 +1719,17 @@ def update(self, **kwargs) -> None: def normalize(self) -> None: """ Normalize weights so each target neuron has sum of connection weights equal to - ``self.norm``. All-zero kernels remain zero. + ``self.norm``. Kernels whose weights sum to zero are left unchanged. """ if self.norm is not None: # get a view and modify in-place # w: ch_in, ch_out * h_out, k w = self.w.view(self.w.shape[0] * self.w.shape[1], self.w.shape[2]) - for fltr in range(w.shape[0]): - total = w[fltr].sum(0).masked_fill(w[fltr].abs().sum(0) == 0, 1) - w[fltr] *= self.norm / total + w_sum = w.sum(1, keepdim=True) + scale = self.norm / w_sum + scale.masked_fill_(w_sum == 0, 1.0) + w *= scale def reset_state_variables(self) -> None: """ @@ -1865,16 +1872,17 @@ def update(self, **kwargs) -> None: def normalize(self) -> None: """ Normalize weights so each target neuron has sum of connection weights equal to - ``self.norm``. All-zero kernels remain zero. + ``self.norm``. Kernels whose weights sum to zero are left unchanged. """ if self.norm is not None: # get a view and modify in-place # w: ch_in, ch_out * w_out * h_out, k1 * k2 w = self.w.view(self.w.shape[0] * self.w.shape[1], self.w.shape[2]) - for fltr in range(w.shape[0]): - total = w[fltr].sum(0).masked_fill(w[fltr].abs().sum(0) == 0, 1) - w[fltr] *= self.norm / total + w_sum = w.sum(1, keepdim=True) + scale = self.norm / w_sum + scale.masked_fill_(w_sum == 0, 1.0) + w *= scale def reset_state_variables(self) -> None: """ @@ -2020,16 +2028,17 @@ def update(self, **kwargs) -> None: def normalize(self) -> None: """ Normalize weights so each target neuron has sum of connection weights equal to - ``self.norm``. All-zero kernels remain zero. + ``self.norm``. Kernels whose weights sum to zero are left unchanged. """ if self.norm is not None: # get a view and modify in-place # w: ch_in, ch_out * w_out * h_out * d_out, k1*k2*k3 w = self.w.view(self.w.shape[0] * self.w.shape[1], self.w.shape[2]) - for fltr in range(w.shape[0]): - total = w[fltr].sum(0).masked_fill(w[fltr].abs().sum(0) == 0, 1) - w[fltr] *= self.norm / total + w_sum = w.sum(1, keepdim=True) + scale = self.norm / w_sum + scale.masked_fill_(w_sum == 0, 1.0) + w *= scale def reset_state_variables(self) -> None: """ diff --git a/test/network/test_perf_equivalence.py b/test/network/test_perf_equivalence.py index ba575e19..342d93b6 100644 --- a/test/network/test_perf_equivalence.py +++ b/test/network/test_perf_equivalence.py @@ -18,6 +18,8 @@ * Neuron models update their state buffers in place (same operations, same order) instead of re-assigning module attributes every step. * ``rank_order`` encoding is vectorised. +* Convolutional and local connections normalize every kernel in one operation + instead of a Python loop over kernels. """ import warnings @@ -53,6 +55,10 @@ ) from bindsnet.network.topology import ( Connection, + Conv1dConnection, + Conv2dConnection, + Conv3dConnection, + LocalConnection, LocalConnection1D, LocalConnection2D, LocalConnection3D, @@ -621,3 +627,77 @@ def test_matches_loop_reference(self): ref[times[i] - 1, i] = 1 assert torch.equal(out, ref.reshape(time, 6, 9)) assert out.dtype == torch.uint8 + + +KERNEL_CONNECTIONS = [ + (Conv1dConnection, 1), + (Conv2dConnection, 2), + (Conv3dConnection, 3), + (LocalConnection1D, 1), + (LocalConnection2D, 2), + (LocalConnection3D, 3), +] + + +class TestKernelNormalize: + """One vectorised normalize per connection replaced a loop over kernels.""" + + @staticmethod + def _devices(): + return ["cpu"] + (["cuda"] if torch.cuda.is_available() else []) + + @staticmethod + def _assert_matches_loop(actual, expected): + # CPU agrees bit for bit. CUDA float32 sums one row and many rows in a + # different order, so it may differ by rounding; report the size. + if torch.equal(actual, expected): + return + assert actual.is_cuda, "CPU result must match the loop bit for bit" + diff = (actual - expected).abs().max().item() + warnings.warn( + f"vectorised normalize differs from the loop by rounding only: " + f"max abs diff {diff:.3e}" + ) + assert torch.allclose(actual, expected, rtol=1e-6, atol=1e-7), diff + + @pytest.mark.parametrize("dtype", [torch.float32, torch.float64]) + @pytest.mark.parametrize(("connection_type", "dimensions"), KERNEL_CONNECTIONS) + def test_matches_loop_reference(self, connection_type, dimensions, dtype): + for device in self._devices(): + torch.manual_seed(0) + connection = connection_type( + Input(shape=(2, *((6,) * dimensions))), + LIFNodes(shape=(3, *((4,) * dimensions))), + kernel_size=3, + stride=1, + n_filters=3, + norm=0.4, + w_dtype=dtype, + ).to(device) + with torch.no_grad(): + connection.w.copy_(torch.rand_like(connection.w) - 0.2) + # Previous loop over kernels. + ref = connection.w.clone() + ref_view = ref.view(ref.shape[0] * ref.shape[1], -1) + for fltr in range(ref_view.shape[0]): + ref_view[fltr] *= 0.4 / ref_view[fltr].sum(0) + parameter = connection.w + connection.normalize() + assert connection.w is parameter + self._assert_matches_loop(connection.w, ref) + + def test_local_connection_matches_reference(self): + for device in self._devices(): + torch.manual_seed(0) + connection = LocalConnection( + Input(shape=(1, 6, 6)), + LIFNodes(shape=(3, 4, 4)), + kernel_size=3, + stride=1, + n_filters=3, + norm=0.4, + ).to(device) + ref = connection.w.clone() + ref *= connection.norm / ref.sum(0).view(1, -1) + connection.normalize() + assert torch.equal(connection.w, ref) diff --git a/test/network/test_zero_kernel_normalization.py b/test/network/test_zero_kernel_normalization.py index 88505dfa..056e2988 100644 --- a/test/network/test_zero_kernel_normalization.py +++ b/test/network/test_zero_kernel_normalization.py @@ -6,6 +6,7 @@ Conv1dConnection, Conv2dConnection, Conv3dConnection, + LocalConnection, LocalConnection1D, LocalConnection2D, LocalConnection3D, @@ -59,13 +60,37 @@ def test_zero_kernel_normalization(connection_type, dimensions, dtype, norm): @pytest.mark.parametrize(("connection_type", "dimensions"), CONNECTIONS) -def test_nonzero_signed_cancellation_is_not_a_zero_kernel(connection_type, dimensions): +def test_zero_sum_signed_kernel_is_left_unchanged(connection_type, dimensions): connection = _connection(connection_type, dimensions, torch.float32, 2.0) with torch.no_grad(): weights = connection.w.reshape(-1, 2**dimensions) weights[0].zero_() weights[0, 0], weights[0, 1] = 1.0, -1.0 + before = connection.w.clone() connection.normalize() actual = connection.w.reshape(-1, 2**dimensions) - assert torch.isposinf(actual[0, 0]) - assert torch.isneginf(actual[0, 1]) + assert torch.equal(actual[0], before.reshape(-1, 2**dimensions)[0]) + assert torch.isfinite(connection.w).all() + + +@pytest.mark.parametrize("norm", [0.0, 2.0]) +def test_local_connection_zero_target_column(norm): + connection = LocalConnection( + Input(shape=(1, 4, 4)), + LIFNodes(shape=(2, 3, 3)), + kernel_size=2, + stride=1, + n_filters=2, + norm=norm, + ) + with torch.no_grad(): + connection.w.copy_( + torch.rand_like(connection.w) * connection.mask.logical_not() + ) + connection.w[:, 0] = 0.0 + expected = connection.w.clone() + expected[:, 1:] *= connection.norm / expected[:, 1:].sum(0) + for _ in range(2): + connection.normalize() + assert torch.isfinite(connection.w).all() + torch.testing.assert_close(connection.w, expected)