diff --git a/neo/rawio/blackrockrawio.py b/neo/rawio/blackrockrawio.py index 83d1a2771..2447b5fa0 100644 --- a/neo/rawio/blackrockrawio.py +++ b/neo/rawio/blackrockrawio.py @@ -1772,30 +1772,42 @@ def _get_event_segment_ids(self, raw_event_data, masks, spec): # consistency check for monotone increasing time stamps # - Use logical comparator (instead of np.diff) to avoid unsigned dtype issues. # - Only consider handled/known event types. - mask_handled = np.any([mask for mask in masks.values()], axis=0) - jump_ids_handled = ( - np.where( - raw_event_data["timestamp"][mask_handled][1:] < raw_event_data["timestamp"][mask_handled][:-1] - )[0] - + 1 - ) - jump_ids = np.where(mask_handled)[0][jump_ids_handled] # jump ids in full set of events (incl. unhandled) - overlap = np.isin(jump_ids, reset_ev_ids) - if not all(overlap): + # - Compare only packets of the same type: the file does not order packets of different + # types (e.g. a comment is stamped when the user starts typing but written when they + # finish, so it can be later than the spikes written after it). + # jumps maps the position of a packet earlier than the previous packet of its type to the + # position of that previous packet, in the full set of events (incl. unhandled). + jumps = {} + for mask in masks.values(): + ids = np.flatnonzero(mask) + ts = raw_event_data["timestamp"][ids] + earlier = np.flatnonzero(ts[1:] < ts[:-1]) + jumps.update(zip(ids[earlier + 1].tolist(), ids[earlier].tolist())) + + # A reset sends the clock of every packet type back, so each type jumps at its first packet + # after it. Count a jump only if the packet it is compared with is in the current segment, + # so that one reset is one boundary. + reset_set = set(reset_ev_ids.tolist()) + boundaries = [] + segment_start = 0 + for position in sorted(reset_set | jumps.keys()): + if position in reset_set or jumps[position] >= segment_start: + boundaries.append(position) + segment_start = position + additional_ids = np.array([b for b in boundaries if b not in reset_set]) + if len(additional_ids): # additional resets occurred without a reset event being stored - additional_ids = jump_ids[np.invert(overlap)] warnings.warn( f"Detected {len(additional_ids)} undocumented segments within " f"nev data after timestamps {additional_ids}." "" ) - reset_ev_ids = sorted(np.unique(np.concatenate((reset_ev_ids, jump_ids)))) event_segment_ids = np.zeros(len(raw_event_data), dtype=int) - for reset_event_id in reset_ev_ids: - event_segment_ids[reset_event_id:] += 1 + event_segment_ids[boundaries] = 1 + event_segment_ids = np.cumsum(event_segment_ids) - self._nb_segment_nev = len(reset_ev_ids) + 1 + self._nb_segment_nev = len(boundaries) + 1 return event_segment_ids else: diff --git a/neo/test/rawiotest/test_blackrockrawio.py b/neo/test/rawiotest/test_blackrockrawio.py index 03c3481ae..26cf6dfed 100644 --- a/neo/test/rawiotest/test_blackrockrawio.py +++ b/neo/test/rawiotest/test_blackrockrawio.py @@ -3,6 +3,7 @@ """ import unittest +import warnings from neo.rawio.blackrockrawio import BlackrockRawIO from neo.test.rawiotest.common_rawio_test import BaseTestRawIO @@ -374,6 +375,30 @@ def test_gap_tolerance_ms_parameter(self): segments_strict = reader_strict.segment_count(0) self.assertEqual(segments_strict, 3) # + def test_nev_segments_ignore_order_between_packet_types(self): + """ + Comments are stamped when the user starts writing but written to the file when they finish, so they + come after spikes with later timestamps. Only a packet earlier than the previous packet of its own + type is a clock reset, and one reset is one segment boundary. + """ + spike, comment = 1, 0xFFFF + # The comments are 2 ticks ahead of the spike before them, later than the spike after them. + no_reset = [(1000, spike), (1002, comment), (1001, spike), (1010, spike), (1012, comment), (1011, spike)] + # A reset sends the clock of every packet type back. + reset = [(5, spike), (7, comment), (6, spike)] + reader = BlackrockRawIO(filename=self.get_local_path("blackrock/blackrock_3_0/file_spec_3_0")) # not parsed + for packets, expected in ((no_reset, [0] * 6), (no_reset + reset, [0] * 6 + [1] * 3)): + raw = np.array( + [(*p, b"") for p in packets], dtype=[("timestamp", "uint64"), ("packet_id", "uint16"), ("value", "S2")] + ) + masks = {"Spikes": raw["packet_id"] == spike, "Comments": raw["packet_id"] == comment} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + segment_ids = reader._get_event_segment_ids(raw, masks, "3.0") + assert_equal(segment_ids, expected) + self.assertEqual(reader._nb_segment_nev, max(expected) + 1) + self.assertEqual(len(caught), max(expected)) # one warning for the reset without a reset comment + if __name__ == "__main__": unittest.main()