Fix bootstrap socket watcher exhaustion

CMakeLists.txt

@@ -192,6 +192,11 @@ install(FILES ${CMAKE_BINARY_DIR}/${TARGET_NAME}.service
 
 # TESTING
 
+enable_testing()
+add_test(NAME dns_poller_watchers
+  COMMAND ${CMAKE_COMMAND} -E env "CC=${CMAKE_C_COMPILER}"
+          sh ${CMAKE_SOURCE_DIR}/tests/unit/run_dns_poller_test.sh)
+
 find_program(
   PYTHON3_EXE
   NAMES "python3"
@@ -202,8 +207,6 @@ if(NOT PYTHON3_EXE)
 else()
   message(STATUS "python3 found: ${PYTHON3_EXE}")
 
-  enable_testing()
-
   # Robot framework tests
   add_test(NAME robot COMMAND ${PYTHON3_EXE} -m robot.run functional_tests.robot
            WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests/robot)

src/dns_poller.c

@@ -13,9 +13,9 @@ static void sock_cb(struct ev_loop __attribute__((unused)) *loop,
 }
 
 static struct ev_io * get_io_event(dns_poller_t *d, int sock) {
-  for (unsigned i = 0; i < d->io_events_count; i++) {
-    if (d->io_events[i].fd == sock) {
-      return &d->io_events[i];
+  for (dns_io_event_t *event = d->io_events; event; event = event->next) {
+    if (event->watcher.fd == sock) {
+      return &event->watcher;
     }
   }
   return NULL;
@@ -27,16 +27,24 @@ static void sock_state_cb(void *data, int fd, int read, int write) {
   struct ev_io *io_event_ptr = get_io_event(d, fd);
   if (io_event_ptr) {
     ev_io_stop(d->loop, io_event_ptr);
-    io_event_ptr->fd = 0;
+    io_event_ptr->fd = ARES_SOCKET_BAD;
     DLOG("Released used io event: %p", io_event_ptr);
   }
   if (!read && !write) {
     return;
   }
   // reserve and start new event on unused slot
-  io_event_ptr = get_io_event(d, 0);
+  io_event_ptr = get_io_event(d, ARES_SOCKET_BAD);
   if (!io_event_ptr) {
-    FLOG("c-ares needed more IO event handler, than the number of provided nameservers: %u", d->io_events_count);
+    // libev retains watcher pointers: allocate stable nodes instead of reallocating.
+    dns_io_event_t *event = calloc(1, sizeof(*event));
+    if (!event) {
+      FLOG("Out of mem allocating DNS socket watcher");
+    }
+    event->next = d->io_events;
+    d->io_events = event;
+    io_event_ptr = &event->watcher;
+    io_event_ptr->data = d;
   }
   DLOG("Reserved new io event: %p", io_event_ptr);
   ev_io_init(io_event_ptr, sock_cb, fd,
@@ -213,6 +221,7 @@ void dns_poller_init(dns_poller_t *d, struct ev_loop *loop,
                      const char *source_addr,
                      const char *hostname,
                      int family, dns_poller_cb cb, void *cb_data) {
+  d->io_events = NULL;
   int r = ares_library_init(ARES_LIB_INIT_ALL);
   if (r != ARES_SUCCESS) {
     FLOG("ares_library_init error: %s", ares_strerror(r));
@@ -247,27 +256,16 @@ void dns_poller_init(dns_poller_t *d, struct ev_loop *loop,
   ev_timer_init(&d->timer, timer_cb, 0, 0);
   d->timer.data = d;
   ev_timer_start(d->loop, &d->timer);
-
-  unsigned nameservers = 1;
-  for (unsigned i = 0; bootstrap_dns[i]; i++) {
-    if (bootstrap_dns[i] == ',') {
-      nameservers++;
-    }
-  }
-  DLOG("Nameservers count: %d", nameservers);
-  d->io_events = (ev_io *)calloc(nameservers, sizeof(ev_io));  // zeroed!
-  if (!d->io_events) {
-    FLOG("Out of mem");
-  }
-  for (unsigned i = 0; i < nameservers; i++) {
-    d->io_events[i].data = d;
-  }
-  d->io_events_count = nameservers;
 }
 
 void dns_poller_cleanup(dns_poller_t *d) {
   ares_destroy(d->ares);
   ev_timer_stop(d->loop, &d->timer);
   ares_library_cleanup();
-  free(d->io_events);
+  while (d->io_events) {
+    dns_io_event_t *event = d->io_events;
+    d->io_events = event->next;
+    ev_io_stop(d->loop, &event->watcher);
+    free(event);
+  }
 }

src/dns_poller.h

@@ -18,6 +18,11 @@
 typedef void (*dns_poller_cb)(const char* hostname, void *data,
                               const char *addr_list);
 
+typedef struct dns_io_event {
+  ev_io watcher;
+  struct dns_io_event *next;
+} dns_io_event_t;
+
 typedef struct {
   ares_channel ares;
   struct ev_loop *loop;
@@ -29,8 +34,7 @@ typedef struct {
   void *cb_data;
 
   ev_timer timer;
-  ev_io *io_events;
-  unsigned io_events_count;
+  dns_io_event_t *io_events;
 } dns_poller_t;
 
 // Initializes c-ares and starts a timer for periodic DNS resolution on the

tests/unit/run_dns_poller_test.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+# Run from any directory; requires a C compiler and c-ares/libev development files.
+set -eu
+cd "$(dirname "$0")/../.."
+binary=$(mktemp)
+trap 'rm -f "$binary"' EXIT HUP INT TERM
+"${CC:-cc}" -g -Wall -Wextra -I src -D__FILENAME__='"dns_poller_test"' \
+    tests/unit/test_dns_poller.c src/logging.c src/ring_buffer.c \
+    -lcares -lev -o "$binary"
+"$binary"

tests/unit/test_dns_poller.c

@@ -0,0 +1,41 @@
+#include <assert.h>
+#include <sys/socket.h>
+#include <unistd.h>
+#include "../../src/dns_poller.c"
+int main(void) {
+  dns_poller_t d = {0};
+  struct ev_loop *loop = ev_loop_new(0);
+  assert(loop);
+  dns_poller_init(&d, loop, "127.0.0.1,127.0.0.2", 60, NULL,
+                  "example.com", AF_INET, NULL, NULL);
+  ev_timer_stop(loop, &d.timer);
+  int fds[40];
+  ev_io *first = NULL;
+  for (int i = 0; i < 40; i++) {
+    fds[i] = socket(AF_INET, SOCK_DGRAM, 0);
+    assert(fds[i] >= 0);
+    sock_state_cb(&d, fds[i], 1, 0);
+    assert(get_io_event(&d, fds[i]));
+    if (!i) first = get_io_event(&d, fds[i]);
+    assert(get_io_event(&d, fds[0]) == first);
+  }
+  ev_io *reused = get_io_event(&d, fds[10]);
+  sock_state_cb(&d, fds[10], 0, 0);
+  assert(!get_io_event(&d, fds[10]));
+  sock_state_cb(&d, fds[10], 1, 1);
+  assert(get_io_event(&d, fds[10]) == reused);
+  for (int i = 0; i < 40; i++) {
+    sock_state_cb(&d, fds[i], 0, 0);
+    close(fds[i]);
+  }
+  // Also exercise cleanup with a registered watcher still active.
+  int remaining = socket(AF_INET, SOCK_DGRAM, 0);
+  assert(remaining >= 0);
+  sock_state_cb(&d, remaining, 1, 0);
+  dns_poller_cleanup(&d);
+  close(remaining);
+  ev_run(loop, EVRUN_NOWAIT);
+  ev_loop_destroy(loop);
+  puts("PASS: 40 sockets / 2 servers; stable addresses; update/reuse/cleanup");
+  return 0;
+}