perf(reactor): serve events queued past the tail threshold first

Makefile

@@ -897,6 +897,7 @@ TEST_TARGETS = test_worker_affinity \
                test_rx_epoll \
                test_reactor_drain_chunk \
                test_reactor_late \
+               test_reactor_tail \
                test_latency_probe \
                test_sstring \
                test_abi_layout \
@@ -1125,6 +1126,14 @@ test_reactor_late: tests/test_reactor_late.c src/sys/io/reactor.c
 	$(CC) $(CFLAGS) -o $@ tests/test_reactor_late.c -pthread
 	./$@
 
+# Tail-elevation heuristic: an event armed past CWIST_REACTOR_TAIL_US is
+# served before younger events of the same batch on both backends, and its
+# callback observes cwist_reactor_dispatch_late(). Runs each knob value in a
+# child process (the knob is cached on first read) on uring and epoll.
+test_reactor_tail: tests/test_reactor_tail.c src/sys/io/reactor.c
+	$(CC) $(CFLAGS) -o $@ tests/test_reactor_tail.c -pthread
+	./$@
+
 # Latency probe test (issue #166): run twice per build, with the probe
 # enabled (recorder must accumulate samples) and disabled (recorder must
 # stay at zero). CWIST_LATENCY_PROBE must be set before reactor creation.

docs/ko/runtime-config.md

@@ -91,6 +91,7 @@
 | `CWIST_REACTOR_ROUND_BUDGET_US` | tuning | 마이크로초, 0 이상 | `2000` | reactor 라운드가 이 시간을 넘기면, 대기 중인 re-arm을 라운드 끝이 아니라 콜백 64개마다 커널에 넘깁니다. 라운드는 끝까지 실행됩니다. `0`이면 한도를 끕니다. |
 | `CWIST_REACTOR_CQ_GRACE_NS` | tuning | 나노초, 0 이상 | `20000` | 커널에서 잠들기 전에 completion ring을 확인하며 도는 시간. `0`이면 돌지 않습니다. |
 | `CWIST_REACTOR_LATE_US` | tuning | 마이크로초, 0 이상 | `3000` | 실험 기능(#319). reactor 라운드가 completion을 수확한 뒤 이 시간을 넘기면, 남은 콜백을 지연 모드로 처리합니다: 콜백마다 re-arm을 바로 커널에 넘기고, 게시된 작업을 콜백마다 비우며, 파이프라인 HTTP/1.1 연결은 양보하지 않고 쌓인 요청을 처리합니다. io_uring 전용. `0`이면 끕니다. |
+| `CWIST_REACTOR_TAIL_US` | tuning | 마이크로초, 0 이상 | `3500` | tail 우선 처리 휴리스틱(io_uring과 epoll 둘 다). 이 시간보다 오래 전에 arm된 이벤트는 디스패치 배치의 맨 앞으로 끌려와 먼저 처리되고, 그 연결은 쌓인 파이프라인을 양보 없이 처리합니다. 느린 라운드가 아니라 느린 이벤트에 반응한다는 점에서 `CWIST_REACTOR_LATE_US`를 보완합니다. `0`이면 끕니다. |
 
 ## 진단
 

docs/runtime-config.md

@@ -91,6 +91,7 @@ Each variable has one status. The status says what v4 promises about it.
 | `CWIST_REACTOR_ROUND_BUDGET_US` | tuning | microseconds ≥ 0 | `2000` | Once a reactor round runs longer than this, parked re-arms are flushed to the kernel every 64 callbacks instead of at the end of the round. The round still runs to completion. `0` disables the bound. |
 | `CWIST_REACTOR_CQ_GRACE_NS` | tuning | nanoseconds ≥ 0 | `20000` | How long the reactor spins on the completion ring before it sleeps in the kernel. `0` disables the spin. |
 | `CWIST_REACTOR_LATE_US` | tuning | microseconds ≥ 0 | `3000` | Experiment (#319). Once a reactor round has run this long since its completions were reaped, the rest of the round runs in late mode: each callback's re-arm is submitted right after it, posted work is drained after each callback, and a pipelined HTTP/1.1 connection serves its buffered requests without yielding. io_uring only. `0` disables. |
+| `CWIST_REACTOR_TAIL_US` | tuning | microseconds ≥ 0 | `3500` | Tail-elevation heuristic (io_uring and epoll). An event armed longer ago than this is pulled to the front of its dispatch batch and served first; its connection then serves its buffered pipeline without yielding. Complements `CWIST_REACTOR_LATE_US`, which reacts to a slow round rather than a slow event. `0` disables. |
 
 ## Diagnostics
 

src/net/http/http.c

@@ -1101,16 +1101,17 @@ static void http_rx_recv_cb(void *conn_ptr, int res) {
     /* res == -EAGAIN: nothing staged; the POLL fallback below resumes the
      * wait without running the handler on an empty stash. */
 
-    /* Latency probe: rx_armed_ns is only written while the probe is enabled,
-     * so a zero value means the probe is off and both clock reads below are
-     * skipped.  Snapshot it now: the serve path re-arms the next RECV and
-     * would overwrite conn->rx_armed_ns before the queue delay is computed. */
-    uint64_t t0 = 0;
+    /* rx_armed_ns is stamped at every arm. Snapshot it now: the serve path
+     * re-arms the next RECV and would overwrite conn->rx_armed_ns before the
+     * queue delay is computed. One clock read at delivery feeds the
+     * tail-elevation heuristic (CWIST_REACTOR_TAIL_US) and, when the probe
+     * is enabled, the queue-delay sample below. */
     uint64_t armed_ns = conn->rx_armed_ns;
+    uint64_t t0 = 0;
     if (armed_ns) {
-        struct timespec ts;
-        clock_gettime(CLOCK_MONOTONIC, &ts);
-        t0 = (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec;
+        t0 = cwist_reactor_now_ns();
+        if (cwist_reactor_event_aged(t0, armed_ns, cwist_reactor_tail_ns()))
+            cwist_reactor_dispatch_age_bypass();
     }
     conn->last_active_sec = cwist_fast_monotonic_sec();
 
@@ -1149,7 +1150,7 @@ static void http_rx_recv_cb(void *conn_ptr, int res) {
     conn->rx_prefers_poll = false;
     http_async_dispatch(conn->fd, conn, conn->handler, conn->user_ctx, conn->reactor);
 
-    if (armed_ns) {
+    if (armed_ns && cwist_reactor_latency_probe_enabled()) {
         uint64_t queue_us = (t0 - armed_ns) / 1000;
         struct timespec ts;
         clock_gettime(CLOCK_MONOTONIC, &ts);

src/sys/app/app.c

@@ -3314,9 +3314,10 @@ cwist_async_action_t cwist_app_http_handler_async(int client_fd, cwist_http_asyn
     uint32_t priority_weight = ((mixed_seed * 16777619U) >> 8) % 100;
 
     /* One pipeline must not monopolize this reactor's other connections,
-     * unless this callback runs in a late round (issue #319 experiment): the
-     * connection has already waited past CWIST_REACTOR_LATE_US, so it serves
-     * its buffered pipeline now instead of yielding behind the others. */
+     * unless this callback runs in a late round (issue #319 experiment) or the
+     * tail heuristic aged its event out of the batch (CWIST_REACTOR_TAIL_US):
+     * the connection has already waited past the threshold, so it serves its
+     * buffered pipeline now instead of yielding behind the others. */
     const bool late = cwist_reactor_dispatch_late();
     const unsigned int yield_batch = app_http_yield_batch();
     const unsigned int turn_max =