From 885c495e5d08f0b5a7b44f3f6afe966c9a0a4b27 Mon Sep 17 00:00:00 2001 From: Chadwick Boulay Date: Wed, 21 Oct 2026 12:32:37 -0400 Subject: [PATCH] Blackrock: detect NEV clock resets per packet type A NEV packet started a new segment whenever its timestamp was earlier than the previous handled packet's, whatever the type. The file does not order packets of different types: a comment is stamped when the user starts writing it but written when they finish, so it can be later than the spikes written after it, and each such comment split the recording. Compare each packet only with the previous packet of its own type. A real reset sends every type's clock back, so a jump counts only if the packet it is compared with is in the current segment: one reset is one boundary. --- neo/rawio/blackrockrawio.py | 42 +++++++++++++++-------- neo/test/rawiotest/test_blackrockrawio.py | 25 ++++++++++++++ 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/neo/rawio/blackrockrawio.py b/neo/rawio/blackrockrawio.py index 83d1a2771..2447b5fa0 100644 --- a/neo/rawio/blackrockrawio.py +++ b/neo/rawio/blackrockrawio.py @@ -1772,30 +1772,42 @@ def _get_event_segment_ids(self, raw_event_data, masks, spec): # consistency check for monotone increasing time stamps # - Use logical comparator (instead of np.diff) to avoid unsigned dtype issues. # - Only consider handled/known event types. - mask_handled = np.any([mask for mask in masks.values()], axis=0) - jump_ids_handled = ( - np.where( - raw_event_data["timestamp"][mask_handled][1:] < raw_event_data["timestamp"][mask_handled][:-1] - )[0] - + 1 - ) - jump_ids = np.where(mask_handled)[0][jump_ids_handled] # jump ids in full set of events (incl. unhandled) - overlap = np.isin(jump_ids, reset_ev_ids) - if not all(overlap): + # - Compare only packets of the same type: the file does not order packets of different + # types (e.g. a comment is stamped when the user starts typing but written when they + # finish, so it can be later than the spikes written after it). + # jumps maps the position of a packet earlier than the previous packet of its type to the + # position of that previous packet, in the full set of events (incl. unhandled). + jumps = {} + for mask in masks.values(): + ids = np.flatnonzero(mask) + ts = raw_event_data["timestamp"][ids] + earlier = np.flatnonzero(ts[1:] < ts[:-1]) + jumps.update(zip(ids[earlier + 1].tolist(), ids[earlier].tolist())) + + # A reset sends the clock of every packet type back, so each type jumps at its first packet + # after it. Count a jump only if the packet it is compared with is in the current segment, + # so that one reset is one boundary. + reset_set = set(reset_ev_ids.tolist()) + boundaries = [] + segment_start = 0 + for position in sorted(reset_set | jumps.keys()): + if position in reset_set or jumps[position] >= segment_start: + boundaries.append(position) + segment_start = position + additional_ids = np.array([b for b in boundaries if b not in reset_set]) + if len(additional_ids): # additional resets occurred without a reset event being stored - additional_ids = jump_ids[np.invert(overlap)] warnings.warn( f"Detected {len(additional_ids)} undocumented segments within " f"nev data after timestamps {additional_ids}." "" ) - reset_ev_ids = sorted(np.unique(np.concatenate((reset_ev_ids, jump_ids)))) event_segment_ids = np.zeros(len(raw_event_data), dtype=int) - for reset_event_id in reset_ev_ids: - event_segment_ids[reset_event_id:] += 1 + event_segment_ids[boundaries] = 1 + event_segment_ids = np.cumsum(event_segment_ids) - self._nb_segment_nev = len(reset_ev_ids) + 1 + self._nb_segment_nev = len(boundaries) + 1 return event_segment_ids else: diff --git a/neo/test/rawiotest/test_blackrockrawio.py b/neo/test/rawiotest/test_blackrockrawio.py index 03c3481ae..26cf6dfed 100644 --- a/neo/test/rawiotest/test_blackrockrawio.py +++ b/neo/test/rawiotest/test_blackrockrawio.py @@ -3,6 +3,7 @@ """ import unittest +import warnings from neo.rawio.blackrockrawio import BlackrockRawIO from neo.test.rawiotest.common_rawio_test import BaseTestRawIO @@ -374,6 +375,30 @@ def test_gap_tolerance_ms_parameter(self): segments_strict = reader_strict.segment_count(0) self.assertEqual(segments_strict, 3) # + def test_nev_segments_ignore_order_between_packet_types(self): + """ + Comments are stamped when the user starts writing but written to the file when they finish, so they + come after spikes with later timestamps. Only a packet earlier than the previous packet of its own + type is a clock reset, and one reset is one segment boundary. + """ + spike, comment = 1, 0xFFFF + # The comments are 2 ticks ahead of the spike before them, later than the spike after them. + no_reset = [(1000, spike), (1002, comment), (1001, spike), (1010, spike), (1012, comment), (1011, spike)] + # A reset sends the clock of every packet type back. + reset = [(5, spike), (7, comment), (6, spike)] + reader = BlackrockRawIO(filename=self.get_local_path("blackrock/blackrock_3_0/file_spec_3_0")) # not parsed + for packets, expected in ((no_reset, [0] * 6), (no_reset + reset, [0] * 6 + [1] * 3)): + raw = np.array( + [(*p, b"") for p in packets], dtype=[("timestamp", "uint64"), ("packet_id", "uint16"), ("value", "S2")] + ) + masks = {"Spikes": raw["packet_id"] == spike, "Comments": raw["packet_id"] == comment} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + segment_ids = reader._get_event_segment_ids(raw, masks, "3.0") + assert_equal(segment_ids, expected) + self.assertEqual(reader._nb_segment_nev, max(expected) + 1) + self.assertEqual(len(caught), max(expected)) # one warning for the reset without a reset comment + if __name__ == "__main__": unittest.main()