Remove footer-based truncation of samples and clock offsets

CHANGELOG.md

@@ -2,6 +2,7 @@
 ### Fixed
 - Fix conversion to `int` for `sample_count` in XDF footer ([#173](https://github.com/xdf-modules/pyxdf/pull/173) by [Clemens Brunner](https://github.com/cbrnr))
 - Scope the log level set by `verbose` to the `load_xdf` call, so it no longer overrides the calling application's logging configuration for the rest of the process ([#175](https://github.com/xdf-modules/pyxdf/pull/175) by [Stefan Appelhoff](https://github.com/sappelhoff))
+- Stop truncating samples beyond the footer `sample_count` and the last clock offset (introduced in [#150](https://github.com/xdf-modules/xdf-Python/pull/150) and [#158](https://github.com/xdf-modules/xdf-Python/pull/158)), which dropped valid samples; garbage clock offsets are handled by clock reset segmentation since [#167](https://github.com/xdf-modules/xdf-Python/pull/167) ([#176](https://github.com/xdf-modules/pyxdf/issues/176) by [Mathieu Scheltienne](https://github.com/mscheltienne))
 
 ## [1.17.5] - 2026-06-15
 ### Changed

src/pyxdf/pyxdf.py

@@ -385,10 +385,6 @@ def load_xdf(
             else:
                 stream.time_series = np.zeros((0, stream.nchns))
 
-    # detect and truncate corrupted clock offsets (pylsl#67 bug)
-    if synchronize_clocks:
-        temp = _truncate_corrupted_offsets(temp, streams)
-
     # perform (fault-tolerant) clock synchronization if requested
     if synchronize_clocks:
         logger.info("  performing clock synchronization...")
@@ -612,155 +608,6 @@ def _segment_clock_diff(diff, thresh_stds, thresh_secs):
     return b_break
 
 
-def _detect_corrupted_clock_offset(
-    clock_times,
-    clock_values,
-    time_thresh=10.0,
-    value_thresh=10.0,
-):
-    """Detect if the last clock offset entry is statistically anomalous.
-
-    Checks whether the last clock offset entry shows anomalous times or values compared
-    to the rest of the sequence. Uses robust statistics (MAD - Median Absolute
-    Deviation) to handle streams with naturally variable clock offsets.
-
-    This function is used by ``_truncate_corrupted_offsets`` as part of the
-    detection logic for the pylsl#67 bug, but does not alone confirm the bug
-    since legitimate clock resets can also cause statistical anomalies.
-
-    Args:
-        clock_times : list of clock offset measurement times.
-        clock_values : list of clock offset values.
-        time_thresh : flag if last time interval exceeds this multiple of the
-            median interval. (default: 10.0)
-        value_thresh : flag if last value's MAD-based z-score exceeds this.
-            (default: 10.0)
-
-    Returns:
-        bool : True if the last clock offset is statistically anomalous.
-
-    See Also:
-        https://github.com/labstreaminglayer/pylsl/issues/67
-    """
-    times = np.asarray(clock_times)
-    values = np.asarray(clock_values)
-
-    # Need at least 3 offsets for meaningful statistics
-    if len(times) < 3:
-        return False
-
-    # Strategy 1: Detect anomalous time interval
-    intervals = np.diff(times)
-    median_interval = np.median(intervals[:-1])
-    last_interval = np.abs(intervals[-1])  # Use abs to catch backward jumps
-
-    if median_interval > 0:
-        time_ratio = last_interval / median_interval
-    else:
-        time_ratio = np.inf if last_interval > 0 else 1.0
-
-    # Strategy 2: Detect anomalous value using MAD (robust to outliers)
-    # MAD = Median Absolute Deviation from the median
-    median_val = np.median(values[:-1])
-    mad = np.median(np.abs(values[:-1] - median_val))
-
-    # Scale MAD to be comparable to standard deviation (for normal distributions)
-    # MAD * 1.4826 ≈ standard deviation
-    # See: https://en.wikipedia.org/wiki/Median_absolute_deviation#Relation_to_standard_deviation
-    if mad > np.finfo(float).eps:
-        value_zscore = np.abs(values[-1] - median_val) / (1.4826 * mad)
-    else:
-        # If MAD is essentially 0 (all values identical), we cannot reliably
-        # detect value anomalies. Rely on time ratio check alone.
-        value_zscore = 0.0
-
-    # Flag as corrupted if EITHER condition is met
-    # (the bug typically causes both, but even one is enough since we already validate
-    # that there is a problem with the sample count as well)
-    return time_ratio > time_thresh or value_zscore > value_thresh
-
-
-def _truncate_corrupted_offsets(temp, streams):
-    """Truncate corrupted clock offsets and associated extra samples.
-
-    The pylsl#67, liblsl#246 bug occurs when an LSL outlet is destroyed while an inlet
-    is still connected, causing an extra sample with potentially garbage clock
-    offset values to be recorded.
-
-    This function requires BOTH conditions to confirm the pylsl#67, liblsl#246 bug:
-
-    1. Extra samples exist (sample count exceeds footer sample_count).
-    2. The last clock offset is statistically anomalous (see
-       ``_detect_corrupted_clock_offset``).
-
-    If both conditions are met, both the extra samples and the corrupted clock
-    offset are truncated.
-
-    Note: Extra samples alone (without corrupted clock offset) are NOT truncated,
-    as the footer sample_count may be incorrect due to race conditions in
-    liblsl/LabRecorder where final samples arrive after the footer is written.
-    Similarly, anomalous clock offsets alone (without extra samples) are NOT
-    truncated, as they may be due to legitimate clock resets.
-
-    Args:
-        temp : dict of StreamData objects indexed by stream ID.
-        streams : dict of stream info dicts indexed by stream ID.
-
-    Returns:
-        dict : the modified temp dictionary.
-
-    See Also:
-        https://github.com/labstreaminglayer/pylsl/issues/67
-        https://github.com/sccn/liblsl/issues/246
-    """
-    for stream_id, stream in temp.items():
-        # First check if there are extra samples - this is the primary evidence
-        # of a potential pylsl#67, liblsl#246 bug.
-        footer = streams.get(stream_id, {}).get("footer", {}).get("info") or {}
-        sample_count_str = footer.get("sample_count", [None])[0]
-        if sample_count_str is None:
-            continue
-        footer_count = int(float(sample_count_str))  # can be with .0 suffix
-
-        if len(stream.time_stamps) <= footer_count:
-            continue  # No extra samples → no evidence of pylsl#67, liblsl#246
-
-        # Extra samples exist - now check if clock offset is also corrupted.
-        # We require BOTH conditions because:
-        # - Extra samples alone could be valid "late" data (footer off-by-1)
-        # - Anomalous clock offsets alone could be legitimate clock resets
-        # - Both together strongly indicate the pylsl#67, liblsl#246 bug
-        clock_corrupted = False
-        if len(stream.clock_times) >= 3:
-            clock_corrupted = _detect_corrupted_clock_offset(
-                stream.clock_times, stream.clock_values
-            )
-
-        if not clock_corrupted:
-            continue  # Extra samples but normal clock offset → keep everything
-
-        # Both conditions met → truncate both
-        logger.warning(
-            "Stream %s: last clock offset is statistically anomalous, "
-            "truncating (see pylsl#67, liblsl#246).",
-            stream_id,
-        )
-        stream.clock_times = stream.clock_times[:-1]
-        stream.clock_values = stream.clock_values[:-1]
-
-        logger.warning(
-            "Stream %s: sample count (%d) exceeds footer sample_count (%d), "
-            "truncating extra samples.",
-            stream_id,
-            len(stream.time_stamps),
-            footer_count,
-        )
-        stream.time_stamps = stream.time_stamps[:footer_count]
-        stream.time_series = stream.time_series[:footer_count]
-
-    return temp
-
-
 def _detect_clock_resets(
     stream,
     time_thresh_stds,

test/test_clock_sync.py

@@ -1,12 +1,16 @@
+from __future__ import annotations
+
 import logging
+from typing import TYPE_CHECKING
 
 import numpy as np
 import pytest
 from mock_data_stream import MockStreamData
 
 from pyxdf.pyxdf import _clock_sync
 
-# No clock resets
+if TYPE_CHECKING:
+    from typing import Literal
 
 
 @pytest.mark.parametrize("n_clock_offsets", list(range(5)))
@@ -609,3 +613,36 @@ def test_sync_clock_corrupt_final_offset(clock_times):
             (0, 9),
         ],
     )
+
+
+@pytest.mark.parametrize("n_garbage_offsets", [1, 2])
+def test_sync_garbage_final_offsets_and_extra_sample(
+    n_garbage_offsets: Literal[1, 2],
+) -> None:
+    """Recording ending as reported in pylsl#67 must sync without dropping samples.
+
+    Destroying an outlet with liblsl <= 1.17.1 while it is recorded appends one extra
+    sample stamped at the outlet destruction time, and the last clock offsets can then
+    be corrupted (e.g. -771221s). LabRecorder stores offsets as ``time = now - offset``,
+    so a corrupted value shifts both the offset time and value. The corrupted offsets
+    must end up in their own clock segment without any samples.
+    """
+    offset = -0.001
+    clock_times = [1000 + 5 * i for i in range(20)]
+    clock_values = [offset] * 20
+    for i in range(n_garbage_offsets):
+        now = clock_times[19] + 5 * (i + 1)
+        clock_times.append(now + 771221.25)
+        clock_values.append(-771221.25)
+    # regular markers plus the extra sample pushed when the outlet was destroyed
+    time_stamps = np.append(np.arange(1000, 1095, 0.5), 1095.008)
+    streams = {
+        1: MockStreamData(
+            time_stamps=time_stamps,
+            clock_times=clock_times,
+            clock_values=clock_values,
+        )
+    }
+    _clock_sync(streams)
+    np.testing.assert_allclose(streams[1].time_stamps, time_stamps + offset, atol=1e-9)
+    np.testing.assert_equal(streams[1].clock_segments, [(0, len(time_stamps) - 1)])