Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 13 additions & 7 deletions driver/internal/eventloop/portable.go
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,15 @@ func (w *portableWorker) readLoop(c *portableConn) {
func (w *portableWorker) UnregisterConn(fd int) error {
w.mu.Lock()
c, ok := w.conns[fd]
if ok && c.conn == nil {
// register's placeholder: a RegisterConn of this number is still
// opening the net.Conn. The number is not registered yet, so it is
// unknown to this call, as on the Linux workers, whose RegisterConn
// inserts the conn in one step. Leave the placeholder: register
// installs the conn over it, and an UnregisterConn issued after
// RegisterConn returned finds that (celeris#890).
ok = false
}
if ok {
delete(w.conns, fd)
}
Expand Down Expand Up @@ -389,14 +398,11 @@ func (w *portableWorker) Write(fd int, data []byte) error {
// still delivering onRecv. No lock is held across the close or the callback.
// The placeholder register puts in the table while it opens the net.Conn
// (conn == nil) has no onClose and no reader, so for it this only marks it
// closing: there is nothing to close and nothing to report. shutdown and
// UnregisterConn both reach it. After shutdown's table swap register sees the
// closing: there is nothing to close and nothing to report. Only shutdown
// reaches it: UnregisterConn treats a placeholder as an unknown number and
// leaves it (celeris#890). After shutdown's table swap register sees the
// worker closed and drops the conn it opened; a failed open drops the
// placeholder itself. An UnregisterConn that finds the placeholder deletes it
// and returns nil, but register's second section checks only the worker's
// closed flag, not that the placeholder is still in the table, so it installs
// the opened conn and starts its reader anyway: that UnregisterConn is lost
// (celeris#890).
// placeholder itself.
func (c *portableConn) shutdownConn(err error) {
c.mu.Lock()
if c.closing {
Expand Down
115 changes: 115 additions & 0 deletions driver/internal/eventloop/portable_unregister_opening_890_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
package eventloop

// celeris#890 (found by the correctness review of PR #1051): an
// UnregisterConn(fd) that ran while RegisterConn(fd, ...) was still opening
// the net.Conn was lost. register had put a placeholder in the table;
// UnregisterConn found it, deleted it and returned nil; register's second
// section checked only the worker's closed flag, so it installed the opened
// conn and started its reader anyway: the conn stayed in the table, onClose
// never fired and the net.Conn was never closed.
//
// The two outcomes the contract allows are tested (the call is linearizable):
// either UnregisterConn ran before the registration (the number is unknown to
// it: engine.ErrUnknownFD, as on the Linux workers, whose RegisterConn also
// inserts the conn atomically) and the registration stands, fully live; or it
// ran after, returned nil and the conn is fully torn down (onClose fired, the
// net.Conn is closed, the table is clean). Never a nil return with the conn
// left registered. No build tag: the portable worker is compiled everywhere.

import (
"context"
"errors"
"net"
"sync/atomic"
"testing"
"time"

"github.com/goceleris/celeris/internal/engine"
)

func TestUnregisterConnWhileRegisterConnIsOpening890(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
pw := newPortableWorker(ctx, 0)
w := &pw
t.Cleanup(func() { _ = w.shutdown() })

const fd = 4242
ours, theirs := net.Pipe()
t.Cleanup(func() { _ = theirs.Close(); _ = ours.Close() })

opening := make(chan struct{})
release := make(chan struct{})
open := func() (net.Conn, error) {
close(opening)
<-release
return ours, nil
}
var closes atomic.Int32
closed := make(chan error, 4)
onClose := func(err error) {
closes.Add(1)
closed <- err
}

type regResult struct {
fd int
err error
}
regDone := make(chan regResult, 1)
go func() {
got, err := w.register(fd, false, open, func([]byte) {}, onClose)
regDone <- regResult{got, err}
}()
<-opening

unregErr := w.UnregisterConn(fd)
close(release)
reg := <-regDone

switch {
case unregErr == nil:
// UnregisterConn took the registration down: onClose must fire, the
// net.Conn must be closed and the table must not hold the number.
select {
case err := <-closed:
if err != nil {
t.Errorf("onClose(%v), want nil after UnregisterConn", err)
}
case <-time.After(2 * time.Second):
t.Fatalf("UnregisterConn returned nil during the open, but the conn was left registered: onClose never fired (register returned fd %d, err %v)", reg.fd, reg.err)
}
_ = theirs.SetReadDeadline(time.Now().Add(2 * time.Second))
if _, err := theirs.Read(make([]byte, 1)); err == nil || errors.Is(err, context.DeadlineExceeded) {
t.Errorf("the net.Conn is still open after UnregisterConn returned nil: peer read err = %v", err)
}
w.mu.RLock()
_, still := w.conns[fd]
w.mu.RUnlock()
if still {
t.Errorf("the table still holds fd %d after UnregisterConn returned nil", fd)
}
case errors.Is(unregErr, engine.ErrUnknownFD):
// UnregisterConn ran before the registration existed: it did nothing,
// and the registration must be intact.
if reg.err != nil {
t.Fatalf("register failed after an UnregisterConn that reported an unknown fd: %v", reg.err)
}
if closes.Load() != 0 {
t.Fatalf("onClose fired %d times for a registration that was never unregistered", closes.Load())
}
if err := w.UnregisterConn(fd); err != nil {
t.Fatalf("second UnregisterConn: %v", err)
}
select {
case <-closed:
case <-time.After(2 * time.Second):
t.Fatal("the registration did not tear down on its own UnregisterConn: onClose never fired")
}
default:
t.Fatalf("UnregisterConn during the open returned %v", unregErr)
}
if got := closes.Load(); got != 1 {
t.Errorf("onClose fired %d times, want 1", got)
}
}
97 changes: 97 additions & 0 deletions driver/internal/eventloop/register_wait.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
package eventloop

import (
"context"
"errors"
"fmt"
"strings"
"time"

"github.com/goceleris/celeris/internal/engine"
)

// registerWaitBudget bounds how long RegisterConnWait waits for a refused
// number to be finalized. A conn is finalized when the engine has cancelled its
// operations and closed its own duplicate of the descriptor, normally within
// microseconds of UnregisterConn; a close that lingers (SO_LINGER) is closed
// off the worker and can take longer (celeris#735), so the wait is bounded
// instead of unlimited.
const registerWaitBudget = 2 * time.Second

// errRegisterWaitBudget is wrapped into the error RegisterConnWait returns
// when registerWaitBudget has passed (declared by the tests commit, used by the
// next).
var errRegisterWaitBudget = errors.New("celeris/eventloop: waited for the number to be finalized, budget used up")

// IsNumberBusy reports whether err is the io_uring engine's refusal of a
// descriptor number whose previous conn has been unregistered but not yet
// finalized: "celeris/iouring: fd already registered".
// [engine.WorkerLoop.UnregisterConn] documents it: the next socket usually
// gets the lowest free number, which is the number of the conn just closed,
// and the engine keeps that number registered until the old conn is finalized.
// It clears by itself, unlike any other RegisterConn error (the number is
// held by an HTTP conn, the worker has shut down, a duplicate registration on
// the epoll engine, whose onClose has run before UnregisterConn returns).
//
// The engine has no sentinel for it, so the text is matched; the tests of
// the drivers register the same number twice on a real io_uring worker to pin
// it.
func IsNumberBusy(err error) bool {
if err == nil {
return false
}
msg := err.Error()
return strings.HasPrefix(msg, "celeris/iouring:") && strings.HasSuffix(msg, "fd already registered")
}

// RegisterConnWait is loop.RegisterConn that waits out the io_uring engine's
// "fd already registered" refusal (see [IsNumberBusy]) instead of returning it:
// it retries with a growing pause until the number is free, ctx is done, or
// registerWaitBudget has passed, and then returns the refusal wrapped together
// with the reason it stopped (ctx.Err(), or errRegisterWaitBudget), so
// errors.Is sees both. Every other error is returned at once, and a failed
// call leaves fd with the caller, as RegisterConn does. The drivers' dials use
// it: without it Pool.Acquire fails under connection churn on the io_uring
// engine (celeris#887 item 28).
//
// It must not run on the worker goroutine of loop (a handler of the server
// whose engine is that loop's, called inline): the number is finalized by that
// worker, which would be asleep in this wait, so the wait runs to its end,
// stalls every connection of that worker for up to registerWaitBudget, and
// returns the refusal anyway. The drivers do not dial there: with WithEngine on
// io_uring, epoll or adaptive their command pools dial direct net.Conns and
// never call RegisterConn. The one public path left is redis
// Client.Subscribe called from a handler, whose PubSub pool still registers
// with the engine; with the default config its handshake already blocks on the
// worker it occupies, so the case that reaches this wait is ForceRESP2 with no
// password and no DB. The engine gives no cheap, reliable way to tell that the
// caller is on its worker, so this is documented, not guarded. The alternative
// that works on any thread, moving the socket to a free number (F_DUPFD) before
// registering it, changes which descriptor the driver owns and needs its own
// review.
func RegisterConnWait(ctx context.Context, loop engine.WorkerLoop, fd int, onRecv func([]byte), onClose func(error)) error {
err := loop.RegisterConn(fd, onRecv, onClose)
if !IsNumberBusy(err) {
return err
}
deadline := time.Now().Add(registerWaitBudget)
pause := 100 * time.Microsecond
for {
t := time.NewTimer(pause)
select {
case <-ctx.Done():
t.Stop()
return fmt.Errorf("%w (gave up waiting: %w)", err, ctx.Err())
case <-t.C:
}
if err = loop.RegisterConn(fd, onRecv, onClose); !IsNumberBusy(err) {
return err
}
if !time.Now().Before(deadline) {
return fmt.Errorf("%w (gave up waiting: %w)", err, errRegisterWaitBudget)
}
if pause < 5*time.Millisecond {
pause *= 2
}
}
}
100 changes: 100 additions & 0 deletions driver/internal/eventloop/register_wait_887_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package eventloop

import (
"context"
"errors"
"fmt"
"testing"
"time"

"github.com/goceleris/celeris/internal/engine"
)

type scriptedLoop struct {
engine.WorkerLoop
errs []error // returned in order; the last repeats
calls int
}

func (l *scriptedLoop) RegisterConn(int, func([]byte), func(error)) error {
i := l.calls
l.calls++
if i >= len(l.errs) {
i = len(l.errs) - 1
}
return l.errs[i]
}

var errBusy = errors.New("celeris/iouring: fd already registered")

func TestIsNumberBusy887(t *testing.T) {
for _, c := range []struct {
err error
want bool
}{
{nil, false},
{errBusy, true},
{errors.New("celeris/iouring: fd 7 is already an HTTP connection"), false},
{errors.New("celeris/epoll: fd 7 is already registered"), false},
{ErrAlreadyRegistered, false}, // a real duplicate of the standalone loop: it does not clear
{fmt.Errorf("celeris/iouring: worker 0 has shut down, cannot register fd 7: %w", errors.New("engine shut down")), false},
} {
if got := IsNumberBusy(c.err); got != c.want {
t.Errorf("IsNumberBusy(%v) = %v, want %v", c.err, got, c.want)
}
}
}

func TestRegisterConnWait887(t *testing.T) {
other := errors.New("boom")
for _, c := range []struct {
name string
errs []error
cancelled bool
wantErr error
wantCalls int
}{
{"free at once", []error{nil}, false, nil, 1},
{"clears after three refusals", []error{errBusy, errBusy, errBusy, nil}, false, nil, 4},
{"other error is final", []error{other}, false, other, 1},
{"other error after a refusal is final", []error{errBusy, other}, false, other, 2},
{"ctx done returns the refusal", []error{errBusy}, true, errBusy, 1},
} {
t.Run(c.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if c.cancelled {
cancel()
}
l := &scriptedLoop{errs: c.errs}
err := RegisterConnWait(ctx, l, 3, nil, nil)
if !errors.Is(err, c.wantErr) {
t.Fatalf("err = %v, want %v", err, c.wantErr)
}
if c.cancelled && !errors.Is(err, context.Canceled) {
t.Fatalf("err = %v does not wrap the ctx error: a caller cannot tell a cancelled wait from a refusal", err)
}
if l.calls != c.wantCalls {
t.Fatalf("RegisterConn calls = %d, want %d", l.calls, c.wantCalls)
}
})
}
}

// A refusal that never clears is bounded by registerWaitBudget, not by ctx.
func TestRegisterConnWaitIsBoundedWithoutACtxDeadline887(t *testing.T) {
l := &scriptedLoop{errs: []error{errBusy}}
start := time.Now()
err := RegisterConnWait(context.Background(), l, 3, nil, nil)
took := time.Since(start)
if !errors.Is(err, errBusy) {
t.Fatalf("err = %v", err)
}
if !errors.Is(err, errRegisterWaitBudget) {
t.Fatalf("err = %v does not say the wait ran out of budget", err)
}
if took < registerWaitBudget || took > registerWaitBudget+time.Second {
t.Fatalf("gave up after %v, want about %v", took, registerWaitBudget)
}
t.Logf("gave up after %v and %d calls", took, l.calls)
}
6 changes: 4 additions & 2 deletions driver/internal/eventloop/teardown_hooks_890_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -209,8 +209,10 @@ func TestShutdownMarksEveryConnClosedBeforeItClosesTheEpollFd890(t *testing.T) {
// the loaded number after it (mutant WAKEATOMIC): FD() already returns -1 once
// the close has run, so no hook placed after the close can order that shape,
// and the race detector reports nothing (the number is an atomic, the write is
// a syscall). Only the probabilistic oracle of
// TestWriteRacingCloseNeverTouchesTheClosedEventfd862 covers it (celeris#890).
// a syscall). That shape is pinned at the source by
// TestWakeGoesThroughSignalOnly890 (wake_source_guard_890_test.go); the
// probabilistic oracle of TestWriteRacingCloseNeverTouchesTheClosedEventfd862
// only backs it (celeris#890).
func TestAWakeAfterShutdownClosedTheEventfdWritesNothing890(t *testing.T) {
t.Cleanup(func() { testHookAfterWakeClose = nil }) // runs last
l, err := New(1)
Expand Down
3 changes: 2 additions & 1 deletion driver/internal/eventloop/wake_close_862_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,8 @@ func c862FillSendBuffer(t *testing.T, fd int) int {
// pinned deterministically by
// TestAWakeAfterShutdownClosedTheEventfdWritesNothing890; a wake that loads
// w.wakeFD.FD() itself, races the close and writes to the loaded number is
// covered only by the oracle above (celeris#890).
// pinned at the source by TestWakeGoesThroughSignalOnly890; the oracle above
// only backs it (celeris#890).
func TestWriteRacingCloseNeverTouchesTheClosedEventfd862(t *testing.T) {
const rounds, writers = 64, 4
one := []byte{'w'}
Expand Down
Loading
Loading