diff --git a/CHANGELOG.md b/CHANGELOG.md index 100b1a67..5179fe8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,9 @@ see the [GitHub releases / tags](https://github.com/BindsNET/bindsnet/releases). - API reference now includes `learning.MCC_learning`, `network.topology_features`, `environment.cue_reward`, `environment.dot_simulator` and `analysis.dotTrace_plotter`, which were missing. +- Normalizing a kernel whose weights sum to zero (for example an all-zero kernel) + in `LocalConnection` and the 1D/2D/3D convolutional and locally connected + connections leaves it unchanged instead of introducing NaNs or infinities. ### Added - `test/repro/test_packaging_consistency.py`: checks that the Dockerfile's Python is in @@ -78,6 +81,10 @@ see the [GitHub releases / tags](https://github.com/BindsNET/bindsnet/releases). - `test/environment/test_gym_environment.py`. ### Changed +- Convolutional and locally connected connections normalize all kernels in one + operation instead of a Python loop (6 to 45 times faster on CPU). Results are + identical on CPU; on CUDA, float32 kernels of 81 or more weights can differ by + float32 rounding (about 1e-7) because the sum is taken in a different order. - CI: one test workflow (`python-app.yml`: job `build` on Python 3.13 plus a 3.11/3.12 matrix, Poetry 2.4.3 with dependency caching, superseded runs cancelled); `pythonpackage.yml` removed (it ran on every push to every branch and its diff --git a/bindsnet/network/topology.py b/bindsnet/network/topology.py index d74d5124..aa43c5cc 100644 --- a/bindsnet/network/topology.py +++ b/bindsnet/network/topology.py @@ -771,14 +771,16 @@ def normalize(self) -> None: # language=rst """ Normalize weights along the first axis according to total weight per target - neuron. + neuron. Kernels whose weights sum to zero are left unchanged. """ if self.norm is not None: # get a view and modify in place w = self.w.view(self.w.shape[0] * self.w.shape[1], self.w.shape[2]) - for fltr in range(w.shape[0]): - w[fltr] *= self.norm / w[fltr].sum(0) + w_sum = w.sum(1, keepdim=True) + scale = self.norm / w_sum + scale.masked_fill_(w_sum == 0, 1.0) + w *= scale def reset_state_variables(self) -> None: # language=rst @@ -930,7 +932,7 @@ def normalize(self) -> None: # language=rst """ Normalize weights along the first axis according to total weight per target - neuron. + neuron. Kernels whose weights sum to zero are left unchanged. """ if self.norm is not None: # get a view and modify in place @@ -938,8 +940,10 @@ def normalize(self) -> None: self.w.shape[0] * self.w.shape[1], self.w.shape[2] * self.w.shape[3] ) - for fltr in range(w.shape[0]): - w[fltr] *= self.norm / w[fltr].sum(0) + w_sum = w.sum(1, keepdim=True) + scale = self.norm / w_sum + scale.masked_fill_(w_sum == 0, 1.0) + w *= scale def reset_state_variables(self) -> None: # language=rst @@ -1110,7 +1114,7 @@ def normalize(self) -> None: # language=rst """ Normalize weights along the first axis according to total weight per target - neuron. + neuron. Kernels whose weights sum to zero are left unchanged. """ if self.norm is not None: # get a view and modify in place @@ -1119,8 +1123,10 @@ def normalize(self) -> None: self.w.shape[2] * self.w.shape[3] * self.w.shape[4], ) - for fltr in range(w.shape[0]): - w[fltr] *= self.norm / w[fltr].sum(0) + w_sum = w.sum(1, keepdim=True) + scale = self.norm / w_sum + scale.masked_fill_(w_sum == 0, 1.0) + w *= scale def reset_state_variables(self) -> None: # language=rst @@ -1579,7 +1585,10 @@ def normalize(self) -> None: """ if self.norm is not None: w = self.w.view(self.source.n, self.target.n) - w *= self.norm / self.w.sum(0).view(1, -1) + w_sum = w.sum(0, keepdim=True) + scale = self.norm / w_sum + scale.masked_fill_(w_sum == 0, 1.0) + w *= scale def reset_state_variables(self) -> None: # language=rst @@ -1710,15 +1719,17 @@ def update(self, **kwargs) -> None: def normalize(self) -> None: """ Normalize weights so each target neuron has sum of connection weights equal to - ``self.norm``. + ``self.norm``. Kernels whose weights sum to zero are left unchanged. """ if self.norm is not None: # get a view and modify in-place # w: ch_in, ch_out * h_out, k w = self.w.view(self.w.shape[0] * self.w.shape[1], self.w.shape[2]) - for fltr in range(w.shape[0]): - w[fltr, :] *= self.norm / w[fltr, :].sum(0) + w_sum = w.sum(1, keepdim=True) + scale = self.norm / w_sum + scale.masked_fill_(w_sum == 0, 1.0) + w *= scale def reset_state_variables(self) -> None: """ @@ -1861,15 +1872,17 @@ def update(self, **kwargs) -> None: def normalize(self) -> None: """ Normalize weights so each target neuron has sum of connection weights equal to - ``self.norm``. + ``self.norm``. Kernels whose weights sum to zero are left unchanged. """ if self.norm is not None: # get a view and modify in-place # w: ch_in, ch_out * w_out * h_out, k1 * k2 w = self.w.view(self.w.shape[0] * self.w.shape[1], self.w.shape[2]) - for fltr in range(w.shape[0]): - w[fltr, :] *= self.norm / w[fltr, :].sum(0) + w_sum = w.sum(1, keepdim=True) + scale = self.norm / w_sum + scale.masked_fill_(w_sum == 0, 1.0) + w *= scale def reset_state_variables(self) -> None: """ @@ -2015,15 +2028,17 @@ def update(self, **kwargs) -> None: def normalize(self) -> None: """ Normalize weights so each target neuron has sum of connection weights equal to - ``self.norm``. + ``self.norm``. Kernels whose weights sum to zero are left unchanged. """ if self.norm is not None: # get a view and modify in-place # w: ch_in, ch_out * w_out * h_out * d_out, k1*k2*k3 w = self.w.view(self.w.shape[0] * self.w.shape[1], self.w.shape[2]) - for fltr in range(w.shape[0]): - w[fltr, :] *= self.norm / w[fltr, :].sum(0) + w_sum = w.sum(1, keepdim=True) + scale = self.norm / w_sum + scale.masked_fill_(w_sum == 0, 1.0) + w *= scale def reset_state_variables(self) -> None: """ diff --git a/test/network/test_perf_equivalence.py b/test/network/test_perf_equivalence.py index ba575e19..342d93b6 100644 --- a/test/network/test_perf_equivalence.py +++ b/test/network/test_perf_equivalence.py @@ -18,6 +18,8 @@ * Neuron models update their state buffers in place (same operations, same order) instead of re-assigning module attributes every step. * ``rank_order`` encoding is vectorised. +* Convolutional and local connections normalize every kernel in one operation + instead of a Python loop over kernels. """ import warnings @@ -53,6 +55,10 @@ ) from bindsnet.network.topology import ( Connection, + Conv1dConnection, + Conv2dConnection, + Conv3dConnection, + LocalConnection, LocalConnection1D, LocalConnection2D, LocalConnection3D, @@ -621,3 +627,77 @@ def test_matches_loop_reference(self): ref[times[i] - 1, i] = 1 assert torch.equal(out, ref.reshape(time, 6, 9)) assert out.dtype == torch.uint8 + + +KERNEL_CONNECTIONS = [ + (Conv1dConnection, 1), + (Conv2dConnection, 2), + (Conv3dConnection, 3), + (LocalConnection1D, 1), + (LocalConnection2D, 2), + (LocalConnection3D, 3), +] + + +class TestKernelNormalize: + """One vectorised normalize per connection replaced a loop over kernels.""" + + @staticmethod + def _devices(): + return ["cpu"] + (["cuda"] if torch.cuda.is_available() else []) + + @staticmethod + def _assert_matches_loop(actual, expected): + # CPU agrees bit for bit. CUDA float32 sums one row and many rows in a + # different order, so it may differ by rounding; report the size. + if torch.equal(actual, expected): + return + assert actual.is_cuda, "CPU result must match the loop bit for bit" + diff = (actual - expected).abs().max().item() + warnings.warn( + f"vectorised normalize differs from the loop by rounding only: " + f"max abs diff {diff:.3e}" + ) + assert torch.allclose(actual, expected, rtol=1e-6, atol=1e-7), diff + + @pytest.mark.parametrize("dtype", [torch.float32, torch.float64]) + @pytest.mark.parametrize(("connection_type", "dimensions"), KERNEL_CONNECTIONS) + def test_matches_loop_reference(self, connection_type, dimensions, dtype): + for device in self._devices(): + torch.manual_seed(0) + connection = connection_type( + Input(shape=(2, *((6,) * dimensions))), + LIFNodes(shape=(3, *((4,) * dimensions))), + kernel_size=3, + stride=1, + n_filters=3, + norm=0.4, + w_dtype=dtype, + ).to(device) + with torch.no_grad(): + connection.w.copy_(torch.rand_like(connection.w) - 0.2) + # Previous loop over kernels. + ref = connection.w.clone() + ref_view = ref.view(ref.shape[0] * ref.shape[1], -1) + for fltr in range(ref_view.shape[0]): + ref_view[fltr] *= 0.4 / ref_view[fltr].sum(0) + parameter = connection.w + connection.normalize() + assert connection.w is parameter + self._assert_matches_loop(connection.w, ref) + + def test_local_connection_matches_reference(self): + for device in self._devices(): + torch.manual_seed(0) + connection = LocalConnection( + Input(shape=(1, 6, 6)), + LIFNodes(shape=(3, 4, 4)), + kernel_size=3, + stride=1, + n_filters=3, + norm=0.4, + ).to(device) + ref = connection.w.clone() + ref *= connection.norm / ref.sum(0).view(1, -1) + connection.normalize() + assert torch.equal(connection.w, ref) diff --git a/test/network/test_zero_kernel_normalization.py b/test/network/test_zero_kernel_normalization.py new file mode 100644 index 00000000..056e2988 --- /dev/null +++ b/test/network/test_zero_kernel_normalization.py @@ -0,0 +1,96 @@ +import pytest +import torch + +from bindsnet.network.nodes import Input, LIFNodes +from bindsnet.network.topology import ( + Conv1dConnection, + Conv2dConnection, + Conv3dConnection, + LocalConnection, + LocalConnection1D, + LocalConnection2D, + LocalConnection3D, +) + +CONNECTIONS = [ + (Conv1dConnection, 1), + (Conv2dConnection, 2), + (Conv3dConnection, 3), + (LocalConnection1D, 1), + (LocalConnection2D, 2), + (LocalConnection3D, 3), +] + + +def _connection(connection_type, dimensions, dtype, norm): + return connection_type( + Input(shape=(1, *((4,) * dimensions))), + LIFNodes(shape=(2, *((3,) * dimensions))), + kernel_size=2, + stride=1, + n_filters=2, + norm=norm, + w_dtype=dtype, + ) + + +@pytest.mark.parametrize(("connection_type", "dimensions"), CONNECTIONS) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float64]) +@pytest.mark.parametrize("norm", [None, 0.0, 2.0]) +def test_zero_kernel_normalization(connection_type, dimensions, dtype, norm): + connection = _connection(connection_type, dimensions, dtype, norm) + kernel_elements = 2**dimensions + rows = connection.w.numel() // kernel_elements + ramp = torch.arange(1, kernel_elements + 1, dtype=dtype) + weights = ramp.repeat(rows, 1) + weights[0].zero_() + with torch.no_grad(): + connection.w.copy_(weights.reshape_as(connection.w)) + parameter = connection.w + expected = weights.clone() + if norm is not None: + expected *= norm / (kernel_elements * (kernel_elements + 1) / 2) + for _ in range(2): + connection.normalize() + assert connection.w is parameter + torch.testing.assert_close(connection.w.reshape(rows, -1), expected) + if dtype == torch.float32: + output = connection.compute(torch.ones((1, 1, *((4,) * dimensions)))) + assert torch.isfinite(output).all() + + +@pytest.mark.parametrize(("connection_type", "dimensions"), CONNECTIONS) +def test_zero_sum_signed_kernel_is_left_unchanged(connection_type, dimensions): + connection = _connection(connection_type, dimensions, torch.float32, 2.0) + with torch.no_grad(): + weights = connection.w.reshape(-1, 2**dimensions) + weights[0].zero_() + weights[0, 0], weights[0, 1] = 1.0, -1.0 + before = connection.w.clone() + connection.normalize() + actual = connection.w.reshape(-1, 2**dimensions) + assert torch.equal(actual[0], before.reshape(-1, 2**dimensions)[0]) + assert torch.isfinite(connection.w).all() + + +@pytest.mark.parametrize("norm", [0.0, 2.0]) +def test_local_connection_zero_target_column(norm): + connection = LocalConnection( + Input(shape=(1, 4, 4)), + LIFNodes(shape=(2, 3, 3)), + kernel_size=2, + stride=1, + n_filters=2, + norm=norm, + ) + with torch.no_grad(): + connection.w.copy_( + torch.rand_like(connection.w) * connection.mask.logical_not() + ) + connection.w[:, 0] = 0.0 + expected = connection.w.clone() + expected[:, 1:] *= connection.norm / expected[:, 1:].sum(0) + for _ in range(2): + connection.normalize() + assert torch.isfinite(connection.w).all() + torch.testing.assert_close(connection.w, expected)