From 03961d8515295ab5b356fc5592cd65badc05c042 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 31 Oct 2026 18:39:18 +0100 Subject: [PATCH 1/9] netstack's lower layers carry several links: ranked interfaces, per-link credit, a lost address's flows ended The P5 build's first half (issues/netstack-drives-one-card.md); the node and netstack follow, and do not build against this commit alone. - link: a hello says its medium, Wired or Wireless; i219 and virtio-net say Wired. OneLink goes: a stack refuses a link Open (its address's is alive) or Full. - ip: an interface is added with a Rank, and every choice between interfaces goes by rank and then arrival, where it went by index alone. A transmit opportunity takes Credit, room per interface, which a count still is for every interface alike; the control queue hands out the oldest frame whose link has room. A UDP datagram may name its interface, and send_udp answers NoRoom for one whose link has none. - udp: a received datagram says the interface it came by; the acquisition socket's send names an Origin, its source and its interface. - tcp: lose_address ends every connection from an address the host gave up, Failure::AddressGone, and sends nothing. - shard: no interface until the shell adds one per link; frames come in and go out with their interface, and a flow whose route leaves by a link with no room waits on that link. A removed address, or one ip lost to another host, ends its TCP connections. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + lib/link/src/driver.rs | 9 +- lib/link/src/wire.rs | 70 ++++-- system/i219/src/nic.rs | 2 + system/netstack/dhcp/tests/common/mod.rs | 8 +- system/netstack/dhcp/tests/stack.rs | 2 +- system/netstack/ip/src/config.rs | 2 +- system/netstack/ip/src/egress.rs | 68 +++++- system/netstack/ip/src/iface.rs | 1 + system/netstack/ip/src/lib.rs | 23 +- system/netstack/ip/src/route.rs | 10 +- system/netstack/ip/tests/clk.rs | 12 +- system/netstack/ip/tests/common/mod.rs | 17 +- system/netstack/ip/tests/rank.rs | 127 +++++++++++ system/netstack/shard/src/lib.rs | 241 +++++++++++++-------- system/netstack/shard/tcp/src/lib.rs | 2 + system/netstack/shard/tcp/src/stack.rs | 16 ++ system/netstack/shard/tcp/tests/close.rs | 15 ++ system/netstack/shard/testnet/Cargo.toml | 1 + system/netstack/shard/testnet/src/lib.rs | 46 +++- system/netstack/shard/tests/acquisition.rs | 55 ++--- system/netstack/shard/tests/common/mod.rs | 6 +- system/netstack/shard/tests/drr.rs | 14 +- system/netstack/shard/tests/egress.rs | 44 ++-- system/netstack/shard/tests/icmp.rs | 2 +- system/netstack/shard/tests/links.rs | 170 +++++++++++++++ system/netstack/shard/tests/log.rs | 12 +- system/netstack/shard/tests/udp.rs | 78 +++---- system/netstack/udp/src/lib.rs | 33 ++- system/netstack/udp/tests/common/mod.rs | 9 +- system/netstack/udp/tests/send.rs | 8 +- system/virtio-net/src/nic.rs | 2 + 32 files changed, 827 insertions(+), 279 deletions(-) create mode 100644 system/netstack/ip/tests/rank.rs create mode 100644 system/netstack/shard/tests/links.rs diff --git a/Cargo.lock b/Cargo.lock index 19f6ddbcba9..60cf8ff4a30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3908,6 +3908,7 @@ version = "0.1.0" dependencies = [ "etherparse", "pcap-file", + "toyos-net-ip", "toyos-net-shard", "toyos-net-tcp", "toyos-net-wire", diff --git a/lib/link/src/driver.rs b/lib/link/src/driver.rs index 4216b88ff89..445fe45b1c4 100644 --- a/lib/link/src/driver.rs +++ b/lib/link/src/driver.rs @@ -27,11 +27,14 @@ use crate::ends::{Driver, Fault}; use crate::layout::{RX_DEPTH, SLOT_BYTES, TX_DEPTH}; use crate::region::Region; use crate::state::{Counters, Group, Link}; -use crate::wire::{self, Address, Hello, Refusal}; +use crate::wire::{self, Address, Hello, Medium, Refusal}; use crate::PORT; /// The device under a link, as its driver drives it. pub trait Nic { + /// What the device's link is carried over. + const MEDIUM: Medium; + fn mac(&self) -> [u8; 6]; /// The claim, for the poller: readable means an interrupt has landed. @@ -131,12 +134,12 @@ struct Session<'a> { impl<'a> Session<'a> { /// A fresh region and its end, said hello with over `conn`. - fn open(label: &'a str, nic: &impl Nic, conn: Connection) -> Self { + fn open(label: &'a str, nic: &N, conn: Connection) -> Self { let region = Region::create().unwrap_or_else(|why| panic!("{label}: no region for a link: {why:?}")); let end = Driver::new(region.words()); let mac = Address::new(nic.mac()) .unwrap_or_else(|| panic!("{label}: the device's address is a group's, which no link carries")); - let hello = Hello { mac, label: label.into() }; + let hello = Hello { mac, medium: N::MEDIUM, label: label.into() }; let shared = region.share().unwrap_or_else(|why| panic!("{label}: the link's region would not share: {why:?}")); // A connection that will not take the hello is one whose stack has // gone: its hang-up is read in the first pass. diff --git a/lib/link/src/wire.rs b/lib/link/src/wire.rs index 7673a8a5591..fe3769dfae3 100644 --- a/lib/link/src/wire.rs +++ b/lib/link/src/wire.rs @@ -35,43 +35,74 @@ pub const MSG_REFUSED: u32 = 3; /// The longest driver name a hello carries. pub const MAX_LABEL: usize = 32; -/// Who a driver is: its device's address, which is an individual one, and its -/// name for the stack's lines: ASCII letters, digits and `-`. +/// What a link is carried over, which the stack ranks links by: wired before +/// wireless. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Medium { + Wired, + Wireless, +} + +impl Medium { + fn encode(self) -> u8 { + match self { + Self::Wired => 1, + Self::Wireless => 2, + } + } + + fn decode(byte: u8) -> Option { + match byte { + 1 => Some(Self::Wired), + 2 => Some(Self::Wireless), + _ => None, + } + } +} + +/// Who a driver is: its device's address, which is an individual one, what +/// the link is carried over, and its name for the stack's lines: ASCII +/// letters, digits and `-`. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Hello { pub mac: Address, + pub medium: Medium, pub label: String, } impl Hello { - pub const MAX_BYTES: usize = 6 + MAX_LABEL; + pub const MAX_BYTES: usize = 7 + MAX_LABEL; pub fn encode(&self) -> Vec { - let mut out = Vec::with_capacity(6 + self.label.len()); + let mut out = Vec::with_capacity(7 + self.label.len()); out.extend_from_slice(&self.mac.bytes()); + out.push(self.medium.encode()); out.extend_from_slice(self.label.as_bytes()); out } /// `None` for bytes [`Self::encode`] does not make: too short, a group's - /// address, or a name that is empty, too long or not one. + /// address, no medium, or a name that is empty, too long or not one. pub fn decode(bytes: &[u8]) -> Option { - let (mac, label) = bytes.split_first_chunk::<6>()?; + let (mac, rest) = bytes.split_first_chunk::<6>()?; let mac = Address::new(*mac)?; + let (&medium, label) = rest.split_first()?; + let medium = Medium::decode(medium)?; let named = |b: &u8| b.is_ascii_alphanumeric() || *b == b'-'; if label.is_empty() || label.len() > MAX_LABEL || !label.iter().all(named) { return None; } - Some(Self { mac, label: String::from(core::str::from_utf8(label).ok()?) }) + Some(Self { mac, medium, label: String::from(core::str::from_utf8(label).ok()?) }) } } /// Why the stack would not open a link. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Refusal { - /// The stack drives one link, and its address is another's, or a link of - /// this address is open and alive (`issues/netstack-drives-one-card.md`). - OneLink, + /// A link of this address is open and alive. + Open, + /// The stack drives the most links it drives. + Full, /// The stack ended this address's last link for what its driver wrote. /// The driver ends on it, so its supervisor's restart budget bounds a /// driver that keeps writing what no driver writes. @@ -83,18 +114,20 @@ pub enum Refusal { impl Refusal { pub fn encode(self) -> [u8; 4] { match self { - Self::OneLink => 1u32, + Self::Open => 1u32, Self::Malformed => 2, Self::Faulted => 3, + Self::Full => 4, } .to_le_bytes() } pub fn decode(bytes: &[u8]) -> Option { match u32::from_le_bytes(bytes.try_into().ok()?) { - 1 => Some(Self::OneLink), + 1 => Some(Self::Open), 2 => Some(Self::Malformed), 3 => Some(Self::Faulted), + 4 => Some(Self::Full), _ => None, } } @@ -110,9 +143,10 @@ mod tests { #[test] fn a_hello_reads_back_as_written() { - let hello = Hello { mac: mac([0x52, 0x54, 0, 0x12, 0x34, 0x56]), label: "virtio-net".into() }; + let hello = Hello { mac: mac([0x52, 0x54, 0, 0x12, 0x34, 0x56]), medium: Medium::Wired, label: "virtio-net".into() }; assert_eq!(Hello::decode(&hello.encode()), Some(hello)); - let longest = Hello { mac: mac([2; 6]), label: "a".repeat(MAX_LABEL) }; + let longest = Hello { mac: mac([2; 6]), medium: Medium::Wireless, label: "a".repeat(MAX_LABEL) }; + assert_eq!(longest.encode().len(), Hello::MAX_BYTES); assert_eq!(Hello::decode(&longest.encode()), Some(longest)); } @@ -120,8 +154,12 @@ mod tests { /// group's address among them: the node is made with the address. #[test] fn a_hello_no_driver_writes_is_refused() { - let named = |mac: [u8; 6], label: &[u8]| [mac.as_slice(), label].concat(); + let named = |mac: [u8; 6], label: &[u8]| [mac.as_slice(), &[1], label].concat(); assert_eq!(Hello::decode(&[2, 0, 0, 0, 0]), None, "short of an address"); + assert_eq!(Hello::decode(&[2, 0, 0, 0, 0, 0]), None, "no medium"); + for medium in [0, 3, 0xff] { + assert_eq!(Hello::decode(&[[2, 0, 0, 0, 0, 0, medium].as_slice(), b"i219"].concat()), None, "medium {medium}"); + } assert_eq!(Hello::decode(&named([0x01, 0x00, 0x5e, 0, 0, 0xfb], b"i219")), None, "a group's address"); assert_eq!(Hello::decode(&named([0xff; 6], b"i219")), None, "the broadcast address"); assert_eq!(Hello::decode(&named([2; 6], &[b'a'; MAX_LABEL + 1])), None, "a name too long"); @@ -133,7 +171,7 @@ mod tests { #[test] fn every_refusal_reads_back_and_no_other_word_does() { - for why in [Refusal::OneLink, Refusal::Malformed, Refusal::Faulted] { + for why in [Refusal::Open, Refusal::Full, Refusal::Malformed, Refusal::Faulted] { assert_eq!(Refusal::decode(&why.encode()), Some(why)); } assert_eq!(Refusal::decode(&0u32.to_le_bytes()), None); diff --git a/system/i219/src/nic.rs b/system/i219/src/nic.rs index 1f724d64346..9910227fab2 100644 --- a/system/i219/src/nic.rs +++ b/system/i219/src/nic.rs @@ -240,6 +240,8 @@ impl Nic { } impl toyos_link::driver::Nic for Nic { + const MEDIUM: toyos_link::wire::Medium = toyos_link::wire::Medium::Wired; + fn mac(&self) -> [u8; 6] { self.mac } diff --git a/system/netstack/dhcp/tests/common/mod.rs b/system/netstack/dhcp/tests/common/mod.rs index 2821031cd3c..7e5a00da060 100644 --- a/system/netstack/dhcp/tests/common/mod.rs +++ b/system/netstack/dhcp/tests/common/mod.rs @@ -248,7 +248,7 @@ pub fn ms(i: Instant) -> u64 { pub fn framed(t: &Transmission) -> Vec { use toyos_net_ip::{Ip, Sent, FRAME}; let mut ip = Ip::new(Instant::from_nanos(0), [0x5a; 16]); - let if0 = ip.add_interface(Instant::from_nanos(0), IndividualMac::new(MAC_A).unwrap()); + let if0 = ip.add_interface(Instant::from_nanos(0), IndividualMac::new(MAC_A).unwrap(), toyos_net_ip::Rank(0)); ip.link_up(Instant::from_nanos(0), if0).unwrap(); ip.add_address(Instant::from_nanos(0), if0, A, 24).unwrap(); let setup = |ip: &mut Ip, until: Instant| { @@ -274,12 +274,12 @@ pub fn framed(t: &Transmission) -> Vec { let socket = udp.bind(&ip, Ipv4Addr::UNSPECIFIED, toyos_net_wire::Port::new(68), || 0).unwrap(); udp.set_acquisition(socket).unwrap(); udp.set_broadcast(socket, true).unwrap(); - udp.send_from(&mut ip, socket, t.source, t.destination, 67, &t.payload).unwrap(); + udp.send_from(&mut ip, socket, toyos_net_udp::Origin { iface: if0, source: t.source }, t.destination, 67, &t.payload).unwrap(); let mut frame = Vec::new(); let sent = udp.serve(toyos_net_udp::Sender::Socket(socket), |out| { let mut buf = [0u8; FRAME]; - match ip.send_udp(now, out, &mut buf) { - Ok(Sent::Frame(n)) => frame = buf[..n].to_vec(), + match ip.send_udp(now, &usize::MAX, out, &mut buf) { + Ok(Sent::Frame { len: n, .. }) => frame = buf[..n].to_vec(), other => panic!("{other:?}"), } toyos_net_udp::Offer::Taken diff --git a/system/netstack/dhcp/tests/stack.rs b/system/netstack/dhcp/tests/stack.rs index f49916b1ac7..6025611461b 100644 --- a/system/netstack/dhcp/tests/stack.rs +++ b/system/netstack/dhcp/tests/stack.rs @@ -57,7 +57,7 @@ impl Stack { /// A's interface up with no address, netstack's acquisition socket on port 68. fn new() -> Self { let mut ip = Ip::new(Instant::from_nanos(0), [0x5a; 16]); - let if0 = ip.add_interface(Instant::from_nanos(0), IndividualMac::new(MAC_A).unwrap()); + let if0 = ip.add_interface(Instant::from_nanos(0), IndividualMac::new(MAC_A).unwrap(), toyos_net_ip::Rank(0)); ip.link_up(Instant::from_nanos(0), if0).unwrap(); let mut udp = Udp::new(); let socket = udp.bind(&ip, Ipv4Addr::UNSPECIFIED, Port::new(68), || 0).unwrap(); diff --git a/system/netstack/ip/src/config.rs b/system/netstack/ip/src/config.rs index bf7360d1dca..ff44b67e451 100644 --- a/system/netstack/ip/src/config.rs +++ b/system/netstack/ip/src/config.rs @@ -121,7 +121,7 @@ impl Ip { /// gateway's for an off-link peer. pub fn advise(&mut self, now: Instant, remote: Ipv4Addr, advice: Advice) { let now = self.clock(now); - let Ok(route) = route::lookup(&self.ifaces, remote, Source::Any, None) else { return }; + let Ok(route) = route::lookup(&self.ifaces, &self.order, remote, Source::Any, None) else { return }; let NextHop::Neighbour(next_hop) = route.next_hop else { return }; let Some((i, mut cx)) = self.split(now, route.iface) else { return }; nud::advise(i, &mut cx, next_hop, advice == Advice::Confirmed); diff --git a/system/netstack/ip/src/egress.rs b/system/netstack/ip/src/egress.rs index dd91fe40f37..84c65ca0586 100644 --- a/system/netstack/ip/src/egress.rs +++ b/system/netstack/ip/src/egress.rs @@ -113,8 +113,10 @@ impl Control { } } - fn pop(&mut self) -> Option { - let item = self.items.pop_front()?; + /// The oldest item whose interface `room` says has room. + fn pop(&mut self, mut room: impl FnMut(IfIndex) -> bool) -> Option { + let at = self.items.iter().position(|item| room(item.iface()))?; + let item = self.items.remove(at)?; self.forget(&item); self.refill(); Some(item) @@ -164,6 +166,33 @@ impl Control { } } +/// The room a transmit opportunity has, interface by interface. A count is +/// room on every interface alike. +pub trait Credit { + /// Whether `iface` takes one frame more. + fn room(&self, iface: IfIndex) -> bool; + /// One frame left by `iface`. + fn spend(&mut self, iface: IfIndex); +} + +impl Credit for usize { + fn room(&self, _: IfIndex) -> bool { + *self > 0 + } + fn spend(&mut self, _: IfIndex) { + *self = self.saturating_sub(1); + } +} + +impl Credit for &mut C { + fn room(&self, iface: IfIndex) -> bool { + (**self).room(iface) + } + fn spend(&mut self, iface: IfIndex) { + (**self).spend(iface); + } +} + /// A UDP datagram a transport hands [ip] at a transmit opportunity. #[derive(Clone, Copy, Debug)] pub struct UdpOut<'a> { @@ -174,16 +203,22 @@ pub struct UdpOut<'a> { /// Whether its socket held the broadcast permission when it accepted the datagram: one /// without it never leaves in a link broadcast, whatever the prefixes have become since. pub broadcast: bool, + /// The interface it may leave by, where its send named one. + pub iface: Option, pub datagram: UdpBuilder<'a>, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Sent { - /// This many bytes of the caller's buffer are the frame: one credit spent. - Frame(usize), + /// This many bytes of the caller's buffer are the frame, for this interface: one of its + /// credit to spend. + Frame { len: usize, iface: IfIndex }, /// Not taken: this next hop's link address is being asked for. [`Event::Resolved`] or /// [`Event::Failed`] names it when that ends, unless the routes change first. Pending(Ipv4Addr), + /// Not taken: its interface has no room in this opportunity. The hop is its next hop, or + /// its destination where it has none on the link. + NoRoom { iface: IfIndex, hop: Ipv4Addr }, } /// Where a datagram goes on the link. @@ -205,15 +240,17 @@ fn build_vec(source: IndividualMac, destination: MacAddr, body: &B } impl Ip { - /// A transmit opportunity with room for `credit` frames: [ip]'s own frames and the datagrams + /// A transmit opportunity with `credit`'s room: [ip]'s own frames and the datagrams /// resolution released, in the order they were queued, each handed to `sink` with its - /// interface. Returns how many left. - pub fn transmit(&mut self, now: Instant, credit: usize, mut sink: impl FnMut(IfIndex, &[u8])) -> usize { + /// interface; one whose interface has no room waits without holding back another's. Returns + /// how many left. + pub fn transmit(&mut self, now: Instant, mut credit: impl Credit, mut sink: impl FnMut(IfIndex, &[u8])) -> usize { let now = self.clock(now); let mut sent = 0usize; - while sent < credit { - let Some(item) = self.control.pop() else { break }; + while let Some(item) = self.control.pop(|iface| credit.room(iface)) { + let iface = item.iface(); if self.emit(now, item, &mut sink) { + credit.spend(iface); sent = sent.saturating_add(1); } } @@ -301,7 +338,7 @@ impl Ip { /// is known, and otherwise not taken, its next hop asked for. The route is the one of this /// moment, so a link broadcast is refused a datagram that does not carry the permission. A /// refusal is counted and the datagram's flow is told it is unreachable. - pub fn send_udp(&mut self, now: Instant, out: &UdpOut<'_>, frame: &mut [u8; FRAME]) -> Result { + pub fn send_udp(&mut self, now: Instant, credit: &impl Credit, out: &UdpOut<'_>, frame: &mut [u8; FRAME]) -> Result { let now = self.clock(now); let flow = Flow { source: out.source, @@ -310,7 +347,14 @@ impl Ip { destination_port: out.datagram.destination, }; let source = if out.source.is_unspecified() { Source::Unspecified } else { Source::Bound(out.source) }; - let sent = self.route(out.destination, source, None).and_then(|route| { + let sent = self.route(out.destination, source, out.iface).and_then(|route| { + if !credit.room(route.iface) { + let hop = match route.next_hop { + NextHop::Neighbour(next_hop) => next_hop, + NextHop::Broadcast | NextHop::Multicast(_) => out.destination, + }; + return Ok(Sent::NoRoom { iface: route.iface, hop }); + } let builder = Ipv4Builder { source: Ipv4Source::new(route.source).map_err(|_| Counter::RouteNoSourceAddress)?, destination: out.destination, @@ -322,7 +366,7 @@ impl Ip { self.fits(&builder)?; let Some((i, mut cx)) = self.split(now, route.iface) else { return Err(Counter::UnknownInterface) }; match hop(i, &mut cx, &route, out.destination, out.broadcast)? { - Hop::To(mac) => build(i.mac, mac, &builder, frame).map(Sent::Frame).ok_or(Counter::IpExceedsMtu), + Hop::To(mac) => build(i.mac, mac, &builder, frame).map(|len| Sent::Frame { len, iface: route.iface }).ok_or(Counter::IpExceedsMtu), Hop::Asked(next_hop) => Ok(Sent::Pending(next_hop)), } }); diff --git a/system/netstack/ip/src/iface.rs b/system/netstack/ip/src/iface.rs index 3d3525557f2..43a235ded14 100644 --- a/system/netstack/ip/src/iface.rs +++ b/system/netstack/ip/src/iface.rs @@ -17,6 +17,7 @@ use crate::{acd, igmp, limits, nud, IfIndex, Ip}; pub(crate) struct Interface { pub mac: IndividualMac, + pub rank: crate::Rank, pub up: bool, /// In the order they were added. pub addresses: Vec
, diff --git a/system/netstack/ip/src/lib.rs b/system/netstack/ip/src/lib.rs index dad217a5965..4f0665505e5 100644 --- a/system/netstack/ip/src/lib.rs +++ b/system/netstack/ip/src/lib.rs @@ -68,7 +68,7 @@ use toyos_net_wire::Port; pub use addr::AddrState; pub use counters::{Counter, Counters, RefusalLog}; -pub use egress::{Sent, UdpOut, FRAME}; +pub use egress::{Credit, Sent, UdpOut, FRAME}; pub use igmp::IgmpMode; pub use limiter::Limiter; pub use nud::{Delaying, Held, Incomplete, Nud, Pending, Probing, Reachable, Released, Stale, Unreachable}; @@ -80,6 +80,11 @@ pub use toyos_net_wire::Instant; #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct IfIndex(usize); +/// Where an interface stands among the others for every choice between them: +/// the lower first, and among equals the one added first. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Rank(pub u8); + /// The IP MTU of every interface: an Ethernet body. pub const MTU: usize = toyos_net_wire::ethernet::MAX_BODY; @@ -315,6 +320,8 @@ pub enum Resolution { pub struct Ip { draws: draw::Draws, ifaces: Vec, + /// Every interface's index, by rank and then by when it was added. + order: Vec, timers: timers::Timers, control: egress::Control, limiter: Limiter, @@ -330,6 +337,7 @@ impl Ip { Self { draws, ifaces: Vec::new(), + order: Vec::new(), timers: timers::Timers::default(), control: egress::Control::default(), limiter, @@ -339,11 +347,15 @@ impl Ip { } } - pub fn add_interface(&mut self, now: Instant, mac: IndividualMac) -> IfIndex { + pub fn add_interface(&mut self, now: Instant, mac: IndividualMac, rank: Rank) -> IfIndex { let now = self.clock(now); let reachable = self.draws.reachable_time(); + let index = self.ifaces.len(); + let at = self.order.iter().position(|&n| self.ifaces.get(n).is_some_and(|i| i.rank > rank)).unwrap_or(self.order.len()); + self.order.insert(at, index); self.ifaces.push(iface::Interface { mac, + rank, up: false, addresses: Vec::new(), gateways: Vec::new(), @@ -356,7 +368,12 @@ impl Ip { acd: acd::Conflicts::default(), igmp: igmp::Igmp::default(), }); - IfIndex(self.ifaces.len().saturating_sub(1)) + IfIndex(index) + } + + /// The link address of `iface`. + pub fn mac(&self, iface: IfIndex) -> Option { + self.ifaces.get(iface.0).map(|i| i.mac) } pub fn counters(&self) -> &Counters { diff --git a/system/netstack/ip/src/route.rs b/system/netstack/ip/src/route.rs index fc05654905b..122758a6895 100644 --- a/system/netstack/ip/src/route.rs +++ b/system/netstack/ip/src/route.rs @@ -2,7 +2,8 @@ //! on-link gateways per interface; longest prefix first, then 169.254/16 on the link itself (RFC //! 3927 §2.6.2), then the active gateway, and no forwarding. A lookup considers only interfaces //! that are up, and with a bound source only the interface holding it (RFC 1122 §3.3.4.2, the -//! strong model). +//! strong model). Between interfaces that tie, the one of lower [`crate::Rank`] wins, and among +//! equal ranks the one added first. use core::net::Ipv4Addr; @@ -40,7 +41,8 @@ pub enum Source { Unspecified, } -pub(crate) fn lookup(ifaces: &[Interface], destination: Ipv4Addr, source: Source, bound: Option) -> Result { +/// `order` is every interface's index, by rank and then arrival. +pub(crate) fn lookup(ifaces: &[Interface], order: &[usize], destination: Ipv4Addr, source: Source, bound: Option) -> Result { let eligible = |index: usize, i: &Interface| { i.up && bound.is_none_or(|b| b.0 == index) @@ -49,7 +51,7 @@ pub(crate) fn lookup(ifaces: &[Interface], destination: Ipv4Addr, source: Source Source::Any | Source::Unspecified => true, } }; - let candidates = || ifaces.iter().enumerate().filter(|&(n, i)| eligible(n, i)); + let candidates = || order.iter().filter_map(|&n| ifaces.get(n).map(|i| (n, i))).filter(|&(n, i)| eligible(n, i)); let pick_source = |i: &Interface, toward: Ipv4Addr| match source { Source::Bound(addr) => Ok(addr), Source::Unspecified => Err(Counter::RouteNoSourceAddress), @@ -141,7 +143,7 @@ pub(crate) fn withdraw_off_link(i: &mut Interface, cx: &mut Cx<'_>) { impl Ip { /// Where a datagram to `destination` goes, or the refusal a transport reports. pub fn route(&mut self, destination: Ipv4Addr, source: Source, iface: Option) -> Result { - lookup(&self.ifaces, destination, source, iface).inspect_err(|&refusal| self.log.count(refusal)) + lookup(&self.ifaces, &self.order, destination, source, iface).inspect_err(|&refusal| self.log.count(refusal)) } /// Installs the interface's gateways, in order of preference (RFC 2132 §3.5). A list with any diff --git a/system/netstack/ip/tests/clk.rs b/system/netstack/ip/tests/clk.rs index 3be58312317..4ca0d4b6f9e 100644 --- a/system/netstack/ip/tests/clk.rs +++ b/system/netstack/ip/tests/clk.rs @@ -3,7 +3,7 @@ mod common; use common::*; -use toyos_net_ip::{limits, AddrState, Counter, Instant, Nud}; +use toyos_net_ip::{limits, AddrState, Counter, Instant, Nud, Rank}; fn incomplete_b() -> H { let mut h = H::fixture_i(); @@ -87,7 +87,7 @@ type Trace = (Vec<(u64, Vec)>, Vec, Vec, std::time::Duration); fn trace(secret: [u8; 16], seed: u64) -> Trace { let mut ip = toyos_net_ip::Ip::new(Instant::from_nanos(0), secret); - let if0 = ip.add_interface(Instant::from_nanos(0), mac(MAC_A)); + let if0 = ip.add_interface(Instant::from_nanos(0), mac(MAC_A), Rank(0)); ip.link_up(Instant::from_nanos(0), if0).unwrap(); let mut rng = Rng(seed); let mut frames = Vec::new(); @@ -118,8 +118,8 @@ fn trace(secret: [u8; 16], seed: u64) -> Trace { destination: toyos_net_wire::Port::new(5001).unwrap(), data: b"x", }; - let out = toyos_net_ip::UdpOut { source: A, destination, ttl: toyos_net_wire::ipv4::Ttl::DEFAULT, broadcast: false, datagram }; - if let Ok(toyos_net_ip::Sent::Frame(n)) = ip.send_udp(at, &out, &mut frame) { + let out = toyos_net_ip::UdpOut { source: A, destination, ttl: toyos_net_wire::ipv4::Ttl::DEFAULT, broadcast: false, iface: None, datagram }; + if let Ok(toyos_net_ip::Sent::Frame { len: n, .. }) = ip.send_udp(at, &usize::MAX, &out, &mut frame) { frames.push((now, frame[..n].to_vec())); } } @@ -223,8 +223,8 @@ fn s_ip_clk_010_a_deadline_saturates() { destination: toyos_net_wire::Port::new(5001).unwrap(), data: b"hi", }; - let out = toyos_net_ip::UdpOut { source: A, destination: B, ttl: toyos_net_wire::ipv4::Ttl::DEFAULT, broadcast: false, datagram }; - assert_eq!(h.ip.send_udp(late, &out, &mut frame), Ok(toyos_net_ip::Sent::Pending(B))); + let out = toyos_net_ip::UdpOut { source: A, destination: B, ttl: toyos_net_wire::ipv4::Ttl::DEFAULT, broadcast: false, iface: None, datagram }; + assert_eq!(h.ip.send_udp(late, &usize::MAX, &out, &mut frame), Ok(toyos_net_ip::Sent::Pending(B))); assert_eq!(h.ip.transmit(late, usize::MAX, |_, _| {}), 1); assert!(matches!(h.state(B), Some(Nud::Incomplete(_)))); assert_eq!(h.ip.next_deadline(), Some(Instant::from_nanos(u64::MAX))); diff --git a/system/netstack/ip/tests/common/mod.rs b/system/netstack/ip/tests/common/mod.rs index 01cb60a2879..dc745401075 100644 --- a/system/netstack/ip/tests/common/mod.rs +++ b/system/netstack/ip/tests/common/mod.rs @@ -7,7 +7,7 @@ use std::net::Ipv4Addr; -use toyos_net_ip::{AddrState, Counter, Event, IfIndex, Instant, Ip, Nud, Sent, UdpOut, FRAME}; +use toyos_net_ip::{AddrState, Counter, Event, IfIndex, Instant, Ip, Nud, Rank, Sent, UdpOut, FRAME}; use toyos_net_wire::ipv4::Ttl; use toyos_net_wire::udp::UdpBuilder; use toyos_net_wire::Port; @@ -217,7 +217,7 @@ impl H { /// Interface `if0` with MAC A and its link up at time 0, nothing settled. pub fn raw() -> Self { let mut ip = Ip::new(Instant::from_nanos(0), SECRET); - let if0 = ip.add_interface(Instant::from_nanos(0), mac(MAC_A)); + let if0 = ip.add_interface(Instant::from_nanos(0), mac(MAC_A), Rank(0)); ip.link_up(Instant::from_nanos(0), if0).unwrap(); Self { ip, if0, now: 0, events: Vec::new(), counters: Vec::new(), wire: std::collections::BTreeMap::default() } } @@ -745,12 +745,13 @@ impl H { pub fn send_as(&mut self, source: Ipv4Addr, destination: Ipv4Addr, sport: u16, dport: u16, data: &[u8], ttl: Ttl, broadcast: bool) -> Result>, Counter> { let mut frame = [0u8; FRAME]; let datagram = UdpBuilder { source: Port::new(sport).unwrap(), destination: Port::new(dport).unwrap(), data }; - let out = UdpOut { source, destination, ttl, broadcast, datagram }; - let sent = self.ip.send_udp(self.clock(), &out, &mut frame); + let out = UdpOut { source, destination, ttl, broadcast, iface: None, datagram }; + let sent = self.ip.send_udp(self.clock(), &usize::MAX, &out, &mut frame); self.collect(); sent.map(|s| match s { - Sent::Frame(n) => Some(frame[..n].to_vec()), + Sent::Frame { len: n, .. } => Some(frame[..n].to_vec()), Sent::Pending(_) => None, + Sent::NoRoom { .. } => unreachable!("every interface has room"), }) } @@ -788,7 +789,7 @@ pub const MAC_A1: MacAddr = MacAddr([2, 0, 0, 0, 0, 0x1a]); impl H { /// A second interface with its own MAC and link up, at setup time 0. pub fn add_if1(&mut self) -> IfIndex { - let if1 = self.ip.add_interface(Instant::from_nanos(0), mac(MAC_A1)); + let if1 = self.ip.add_interface(Instant::from_nanos(0), mac(MAC_A1), Rank(0)); self.ip.link_up(Instant::from_nanos(0), if1).unwrap(); if1 } @@ -796,7 +797,7 @@ impl H { /// ToyOS as B: MAC B, 192.0.2.2/24, A resolved at setup. pub fn as_b() -> Self { let mut ip = Ip::new(Instant::from_nanos(0), SECRET); - let if0 = ip.add_interface(Instant::from_nanos(0), mac(MAC_B)); + let if0 = ip.add_interface(Instant::from_nanos(0), mac(MAC_B), Rank(0)); ip.link_up(Instant::from_nanos(0), if0).unwrap(); let mut h = Self { ip, if0, now: 0, events: Vec::new(), counters: Vec::new(), wire: std::collections::BTreeMap::default() }; h.assign(if0, B, 24, 0); @@ -826,7 +827,7 @@ impl H { /// A host with MAC `m` holding `addr/len`, assigned through conflict detection, nothing learned. pub fn host(m: MacAddr, addr: Ipv4Addr, len: u8) -> Self { let mut ip = Ip::new(Instant::from_nanos(0), SECRET); - let if0 = ip.add_interface(Instant::from_nanos(0), mac(m)); + let if0 = ip.add_interface(Instant::from_nanos(0), mac(m), Rank(0)); ip.link_up(Instant::from_nanos(0), if0).unwrap(); let mut h = Self { ip, if0, now: 0, events: Vec::new(), counters: Vec::new(), wire: std::collections::BTreeMap::default() }; h.assign(if0, addr, len, 0); diff --git a/system/netstack/ip/tests/rank.rs b/system/netstack/ip/tests/rank.rs new file mode 100644 index 00000000000..fb0da48d8d9 --- /dev/null +++ b/system/netstack/ip/tests/rank.rs @@ -0,0 +1,127 @@ +//! Interfaces that tie are chosen by rank, then by when each was added: the +//! default route, failover to the next as one goes down and its return. + +mod common; + +use std::net::Ipv4Addr; + +use common::*; +use toyos_net_ip::{Counter, IfIndex, Instant, Ip, NextHop, Rank, Route, Source}; +use toyos_net_wire::ethernet::MacAddr; + +const WIRED: Rank = Rank(0); +const WIRELESS: Rank = Rank(1); + +/// One interface's address and its gateway, on a /24 of its own. +struct Net { + mac: MacAddr, + addr: Ipv4Addr, + gateway: Ipv4Addr, +} + +const NET_1: Net = Net { mac: MacAddr([2, 0, 0, 0, 1, 1]), addr: Ipv4Addr::new(192, 0, 2, 10), gateway: Ipv4Addr::new(192, 0, 2, 1) }; +const NET_2: Net = Net { mac: MacAddr([2, 0, 0, 0, 2, 2]), addr: Ipv4Addr::new(198, 51, 100, 10), gateway: Ipv4Addr::new(198, 51, 100, 1) }; +const NET_3: Net = Net { mac: MacAddr([2, 0, 0, 0, 3, 3]), addr: Ipv4Addr::new(203, 0, 113, 10), gateway: Ipv4Addr::new(203, 0, 113, 1) }; + +/// Somewhere only a gateway reaches. +const OFF_LINK: Ipv4Addr = Ipv4Addr::new(192, 88, 99, 1); + +/// Each net's interface added in turn with its rank, up, addressed and given +/// its gateway. +fn machine(nets: &[(&Net, Rank)]) -> (Ip, Vec, Instant) { + let mut ip = Ip::new(Instant::from_nanos(0), SECRET); + let mut now = Instant::from_millis(0); + let mut ifaces = Vec::new(); + for &(net, rank) in nets { + let iface = ip.add_interface(now, mac(net.mac), rank); + ip.link_up(now, iface).unwrap(); + ip.add_address(now, iface, net.addr, 24).unwrap(); + ifaces.push(iface); + } + let settled = Instant::from_millis(10_000); + while let Some(deadline) = ip.next_deadline().filter(|d| *d <= settled) { + ip.fire(deadline); + ip.transmit(deadline, usize::MAX, |_, _| {}); + now = deadline; + } + for (&(net, _), &iface) in nets.iter().zip(&ifaces) { + ip.set_gateways(now, iface, &[net.gateway]).unwrap(); + } + (ip, ifaces, settled) +} + +fn via(ip: &mut Ip, destination: Ipv4Addr) -> Result { + ip.route(destination, Source::Any, None) +} + +fn by(iface: IfIndex, net: &Net) -> Result { + Ok(Route { iface, next_hop: NextHop::Neighbour(net.gateway), source: net.addr }) +} + +#[test] +fn the_default_route_leaves_by_the_lower_rank_whichever_came_first() { + let (mut ip, ifaces, _) = machine(&[(&NET_1, WIRELESS), (&NET_2, WIRED)]); + assert_eq!(via(&mut ip, OFF_LINK), by(ifaces[1], &NET_2)); +} + +#[test] +fn among_equal_ranks_the_first_added_leads() { + let (mut ip, ifaces, _) = machine(&[(&NET_1, WIRELESS), (&NET_2, WIRED), (&NET_3, WIRED)]); + assert_eq!(via(&mut ip, OFF_LINK), by(ifaces[1], &NET_2)); + let (mut ip, ifaces, _) = machine(&[(&NET_3, WIRED), (&NET_2, WIRED)]); + assert_eq!(via(&mut ip, OFF_LINK), by(ifaces[0], &NET_3)); +} + +#[test] +fn a_link_that_goes_down_hands_the_default_route_on_and_takes_it_back_on_return() { + let (mut ip, ifaces, now) = machine(&[(&NET_1, WIRELESS), (&NET_2, WIRED)]); + ip.link_down(now, ifaces[1]).unwrap(); + assert_eq!(via(&mut ip, OFF_LINK), by(ifaces[0], &NET_1), "failover to the wireless link"); + ip.link_up(now, ifaces[1]).unwrap(); + assert_eq!(via(&mut ip, OFF_LINK), by(ifaces[1], &NET_2), "the wired link's return"); + ip.link_down(now, ifaces[0]).unwrap(); + ip.link_down(now, ifaces[1]).unwrap(); + assert_eq!(via(&mut ip, OFF_LINK), Err(Counter::RouteNoSourceAddress), "no link up"); +} + +#[test] +fn a_connected_prefix_beats_a_lower_ranks_gateway() { + let (mut ip, ifaces, _) = machine(&[(&NET_1, WIRELESS), (&NET_2, WIRED)]); + let neighbour = Ipv4Addr::new(192, 0, 2, 77); + assert_eq!(via(&mut ip, neighbour), Ok(Route { iface: ifaces[0], next_hop: NextHop::Neighbour(neighbour), source: NET_1.addr })); +} + +#[test] +fn a_bound_source_keeps_its_own_interface_whatever_its_rank() { + let (mut ip, ifaces, now) = machine(&[(&NET_1, WIRELESS), (&NET_2, WIRED)]); + assert_eq!(ip.route(OFF_LINK, Source::Bound(NET_1.addr), None), by(ifaces[0], &NET_1)); + // Its link down, the flow has no route, and does not move to the other address. + ip.link_down(now, ifaces[0]).unwrap(); + assert_eq!(ip.route(OFF_LINK, Source::Bound(NET_1.addr), None), Err(Counter::RouteNoSourceAddress)); +} + +/// Room on one interface alone. +struct Only(IfIndex, usize); + +impl toyos_net_ip::Credit for Only { + fn room(&self, iface: IfIndex) -> bool { + iface == self.0 && self.1 > 0 + } + fn spend(&mut self, _: IfIndex) { + self.1 -= 1; + } +} + +#[test] +fn a_link_without_room_holds_back_no_other_links_frames() { + let (mut ip, ifaces, now) = machine(&[(&NET_1, WIRED), (&NET_2, WIRED)]); + // An ARP request queued on each, the first's ahead. + ip.resolve(now, ifaces[0], NET_1.gateway, NET_1.addr); + ip.resolve(now, ifaces[1], NET_2.gateway, NET_2.addr); + let mut out = Vec::new(); + assert_eq!(ip.transmit(now, Only(ifaces[1], 8), |iface, _| out.push(iface)), 1); + assert_eq!(out, [ifaces[1]], "the second link's request, past the first's"); + out.clear(); + assert_eq!(ip.transmit(now, 8usize, |iface, _| out.push(iface)), 1); + assert_eq!(out, [ifaces[0]], "the first's, still queued"); +} diff --git a/system/netstack/shard/src/lib.rs b/system/netstack/shard/src/lib.rs index a5df372ac4d..1fa84ae49ee 100644 --- a/system/netstack/shard/src/lib.rs +++ b/system/netstack/shard/src/lib.rs @@ -1,11 +1,13 @@ -//! One shard of ToyOS's stack: `toyos-net-ip`, `toyos-net-tcp` and `toyos-net-udp` over one -//! interface. Frames come in through [`Shard::receive`] and leave only when the device pulls them -//! with [`Shard::transmit`]; the caller hands in the time and the secrets, and nothing here reads -//! a clock, draws randomness or does I/O. +//! One shard of ToyOS's stack: `toyos-net-ip`, `toyos-net-tcp` and `toyos-net-udp` over every +//! interface the caller adds ([`Shard::add_interface`]), one per link. Frames come in through +//! [`Shard::receive`] and leave only when the links pull them with [`Shard::transmit`]; the +//! caller hands in the time and the secrets, and nothing here reads a clock, draws randomness or +//! does I/O. //! -//! **Egress.** Each frame of credit goes to [ip]'s own frames first — ARP, IGMP, ICMP and the -//! datagrams resolution released — then to what [tcp] owes outside a connection, -//! and otherwise to the round: a deficit round-robin in bytes over every TCP connection and UDP +//! **Egress.** One opportunity serves every link with room, each frame handed out with the +//! interface it leaves by and spending that link's credit. A frame of credit goes to [ip]'s own +//! frames first — ARP, IGMP, ICMP and the datagrams resolution released — then to what [tcp] +//! owes outside a connection, and otherwise to the round: a deficit round-robin in bytes over every TCP connection and UDP //! sender with something to send, in the order each became eligible (RFC 8290 §3, without its new-flows list). A flow //! waiting for its next hop leaves the round, and rejoins at its tail when woken. A TCP segment is //! built only once its next hop's link address is known, and committed only once its frame is @@ -17,7 +19,9 @@ //! spending nothing and charged nothing, while the sender's datagrams to other next hops leave: //! it is handed to [ip] again when [ip] reports that hop resolved or the routes changed, and //! dropped by [udp] when [ip] reports the hop failed. One [ip] refuses, for want of a route or -//! for a next hop it holds failed or has no room for, is [ip]'s to count and report. +//! for a next hop it holds failed or has no room for, is [ip]'s to count and report. A flow +//! whose route leaves by a link with no room waits on that link as on a next hop, and asks again +//! at the next opportunity that link has room in. //! //! **Refusals.** Each crate's refusals of legacy or insecure input pass through that crate's //! `RefusalLog` here: at most one [`Event::Refused`] per rule in any 10 s, carrying how many @@ -47,7 +51,7 @@ use alloc::collections::{BTreeMap, BTreeSet, VecDeque}; use alloc::vec::Vec; use core::net::Ipv4Addr; -use toyos_net_ip::{Advice, Delivery, ErrorKind, IfIndex, Ip, Limiter, NextHop, Nud, Resolution, Sent, Source, Transport, TransportError, FRAME}; +use toyos_net_ip::{Advice, Delivery, ErrorKind, Ip, Limiter, NextHop, Nud, Resolution, Sent, Source, Transport, TransportError, FRAME}; use toyos_net_tcp::{ConnId, Endpoint, Hop, IcmpError, IcmpKind, Lent, ListenerId, Outgoing, Received, Seq, Served, Status, Tcp, Tuple}; use toyos_net_udp::{Offer, Sender, SocketId, Udp, Verdict}; use toyos_net_udp::Served as UdpServed; @@ -56,6 +60,9 @@ use toyos_net_wire::ipv4::{Ipv4Builder, Ipv4Source, MulticastAddr, TrafficClass, use toyos_net_wire::siphash::Key; use toyos_net_wire::{Instant, Port}; +pub use toyos_net_ip::{Credit, IfIndex, Rank}; +pub use toyos_net_udp::Origin; + /// Independent 128-bit keys from the system's randomness: one leaked value predicts no other. #[derive(Clone, Debug)] pub struct Secrets { @@ -67,7 +74,6 @@ pub struct Secrets { #[derive(Clone, Debug)] pub struct Config { - pub mac: IndividualMac, pub receive_buffer: u32, pub send_buffer: u32, pub secrets: Secrets, @@ -86,10 +92,11 @@ pub enum Refusal { pub enum Event { /// A line for the log, and how many refusals of its rule it stands for beyond itself. Refused { refusal: Refusal, suppressed: u64 }, - Verified(Ipv4Addr), - Conflict { addr: Ipv4Addr, mac: MacAddr }, - Lost { addr: Ipv4Addr, mac: MacAddr }, - NotVerified(Ipv4Addr), + Verified { iface: IfIndex, addr: Ipv4Addr }, + Conflict { iface: IfIndex, addr: Ipv4Addr, mac: MacAddr }, + /// Lost to another host after one defence and removed; every TCP connection from it ended. + Lost { iface: IfIndex, addr: Ipv4Addr, mac: MacAddr }, + NotVerified { iface: IfIndex, addr: Ipv4Addr }, } /// Why an active open was refused. @@ -118,15 +125,20 @@ const TCP_MTU: u16 = { u16::from_le_bytes([low, high]) }; -/// What a waiting flow waits on: its next hop's entry, or room in a full neighbour table. +/// What a waiting flow waits on: its next hop's entry, room in a full neighbour table, or room +/// on the link its route leaves by. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] enum Wait { Hop(Ipv4Addr), Room, + Link(IfIndex), } /// A TCP segment's way out, as the hop question answered it. struct Via { + iface: IfIndex, + /// `iface`'s own link address. + source: IndividualMac, next_hop: Ipv4Addr, mac: MacAddr, } @@ -171,8 +183,8 @@ struct Log { pub struct Shard { ip: Ip, - iface: IfIndex, - mac: IndividualMac, + /// Every interface added, in the order it was. + ifaces: Vec, tcp: Tcp, udp: Udp, resets: Limiter, @@ -184,17 +196,17 @@ pub struct Shard { turn: bool, frame: Box<[u8; FRAME]>, /// The remote addresses of the TCP flows each wait holds, each until [ip] reports a change - /// for it; a flow with no route waits on the routes. + /// for it, and under a link's wait the next hops of the UDP datagrams too; a flow with no + /// route waits on the routes. waiting: BTreeMap>, /// [ip]'s routing generation when the waiting flows last asked. routes: u64, } impl Shard { - /// One interface, link down and with no address: the shell configures it. + /// No interface: the shell adds one per link. pub fn new(now: Instant, config: Config) -> Result { - let mut ip = Ip::new(now, config.secrets.ip); - let iface = ip.add_interface(now, config.mac); + let ip = Ip::new(now, config.secrets.ip); let tcp = Tcp::new(toyos_net_tcp::Config { mtu: TCP_MTU, receive_buffer: config.receive_buffer, @@ -204,8 +216,7 @@ impl Shard { Ok(Self { routes: ip.generation(), ip, - iface, - mac: config.mac, + ifaces: Vec::new(), tcp, udp: Udp::new(), resets: Limiter::new(config.secrets.resets), @@ -222,9 +233,11 @@ impl Shard { &self.ip } - /// The interface the shard's [`Ip`] names. - pub fn iface(&self) -> IfIndex { - self.iface + /// A link's interface, down and with no address: the shell configures it. + pub fn add_interface(&mut self, now: Instant, mac: IndividualMac, rank: Rank) -> IfIndex { + let iface = self.ip.add_interface(now, mac, rank); + self.ifaces.push(iface); + iface } pub fn tcp_counters(&self) -> &toyos_net_tcp::Counters { @@ -242,9 +255,9 @@ impl Shard { // ---- the device ---- - /// A frame the device received. - pub fn receive(&mut self, now: Instant, frame: &[u8]) { - match self.ip.receive(now, self.iface, frame) { + /// A frame `iface`'s link received. + pub fn receive(&mut self, now: Instant, iface: IfIndex, frame: &[u8]) { + match self.ip.receive(now, iface, frame) { Some(Delivery::Tcp(arrival, segment)) => { let resets = &mut self.resets; let (from, to) = (arrival.packet.source(), arrival.packet.destination()); @@ -264,20 +277,37 @@ impl Shard { self.settle(now); } - /// A transmit opportunity with room for `credit` frames, each handed to `sink` as it is built. - /// Returns how many left. - pub fn transmit(&mut self, now: Instant, credit: usize, mut sink: impl FnMut(&[u8])) -> usize { + /// A transmit opportunity with `credit`'s room on each link, each frame handed to `sink` as + /// it is built, with the interface it leaves by. Returns how many left. + pub fn transmit(&mut self, now: Instant, mut credit: impl Credit, mut sink: impl FnMut(IfIndex, &[u8])) -> usize { + let credit = &mut credit; + let room: Vec = self.ifaces.iter().copied().filter(|&iface| credit.room(iface)).collect(); + for iface in room { + for hop in self.waiting.remove(&Wait::Link(iface)).into_iter().flatten() { + self.tcp.wake(hop); + self.udp.wake(hop); + } + } + self.settle(now); let mut spent = 0usize; - while spent < credit { - let sent = self.ip.transmit(now, 1, |_, frame| sink(frame)) > 0 - || self.tcp_frame(now, &mut sink, |tcp, hop, out| tcp.transmit_owed(now, 1, hop, out)).1.is_some() - || self.round_frame(now, &mut sink) - // A flow that waits has queued the request it waits on. - || self.ip.transmit(now, 1, |_, frame| sink(frame)) > 0; - if !sent { + while self.ifaces.iter().any(|&iface| credit.room(iface)) { + let own = self.ip.transmit(now, &mut *credit, &mut sink); + spent = spent.saturating_add(own); + let sent = own > 0 + || self.tcp_frame(now, credit, &mut sink, |tcp, hop, out| tcp.transmit_owed(now, 1, hop, out)).1.is_some() + || self.round_frame(now, credit, &mut sink); + if sent && own == 0 { + spent = spent.saturating_add(1); + } + if sent { + continue; + } + // A flow that waits has queued the request it waits on. + let requests = self.ip.transmit(now, &mut *credit, &mut sink); + spent = spent.saturating_add(requests); + if requests == 0 { break; } - spent = spent.saturating_add(1); } self.settle(now); spent @@ -288,7 +318,7 @@ impl Shard { /// deficit is above 0, each frame's length charged as it leaves, and then goes to the tail /// keeping its deficit. A flow with nothing left leaves the round, its deficit with it: a UDP /// sender as its last datagram is handed out, a TCP connection once served with nothing due. - fn round_frame(&mut self, now: Instant, sink: &mut impl FnMut(&[u8])) -> bool { + fn round_frame(&mut self, now: Instant, credit: &mut impl Credit, sink: &mut impl FnMut(IfIndex, &[u8])) -> bool { loop { let Some(head) = self.round.front_mut() else { return false }; if !core::mem::replace(&mut self.turn, true) { @@ -296,12 +326,12 @@ impl Shard { } let flow = head.flow; let Outcome { frame, leaves } = match flow { - Flow::Tcp(id) => match self.tcp_frame(now, sink, |tcp, hop, out| tcp.serve(now, id, hop, out)) { + Flow::Tcp(id) => match self.tcp_frame(now, credit, sink, |tcp, hop, out| tcp.serve(now, id, hop, out)) { (Served::Sent, Some(len)) => Outcome { frame: Some(len), leaves: false }, (Served::Done, None) => Outcome { frame: None, leaves: true }, (Served::Sent, None) | (Served::Done, Some(_)) => unreachable!("[tcp] answers Sent exactly when the sink framed a segment"), }, - Flow::Udp(sender) => self.udp_frame(now, sender, sink), + Flow::Udp(sender) => self.udp_frame(now, sender, credit, sink), }; if let (Some(len), Some(head)) = (frame, self.round.front_mut()) { head.deficit = head.deficit.saturating_sub(len); @@ -325,18 +355,20 @@ impl Shard { fn tcp_frame( &mut self, now: Instant, - sink: &mut impl FnMut(&[u8]), + credit: &mut impl Credit, + sink: &mut impl FnMut(IfIndex, &[u8]), send: impl FnOnce(&mut Tcp, &mut dyn FnMut(&Tuple) -> Hop, &mut dyn FnMut(&Outgoing<'_>, Via)) -> R, ) -> (R, Option) { - let Self { ip, iface, mac, tcp, frame, waiting, .. } = self; - let iface = *iface; + let Self { ip, tcp, frame, waiting, .. } = self; let mut built = None; - let done = send(tcp, &mut |tuple| hop(ip, now, iface, tuple, waiting), &mut |out, via| { - let Some(bytes) = tcp_datagram(out, *mac, via.mac, frame) else { unreachable!("TCP's MTU, [tcp]'s bounds and [ip]'s addresses fit every segment in a frame") }; - sink(bytes); - built = Some((via.next_hop, out.source, charge(bytes))); + let room = &*credit; + let done = send(tcp, &mut |tuple| hop(ip, now, room, tuple, waiting), &mut |out, via| { + let Some(bytes) = tcp_datagram(out, via.source, via.mac, frame) else { unreachable!("TCP's MTU, [tcp]'s bounds and [ip]'s addresses fit every segment in a frame") }; + sink(via.iface, bytes); + built = Some((via.iface, via.next_hop, out.source, charge(bytes))); }); - let Some((next_hop, source, len)) = built else { return (done, None) }; + let Some((iface, next_hop, source, len)) = built else { return (done, None) }; + credit.spend(iface); // The send the neighbour machine counts: STALE moves to DELAY (RFC 4861 §7.3.3). ip.resolve(now, iface, next_hop, source); (done, Some(len)) @@ -344,22 +376,30 @@ impl Shard { /// One datagram of `sender`'s, framed by [ip]: the oldest whose next hop's link address is /// known, or that needs none. One [ip] refuses as it leaves makes no frame. - fn udp_frame(&mut self, now: Instant, sender: Sender, sink: &mut impl FnMut(&[u8])) -> Outcome { - let Self { ip, udp, frame, .. } = self; + fn udp_frame(&mut self, now: Instant, sender: Sender, credit: &mut impl Credit, sink: &mut impl FnMut(IfIndex, &[u8])) -> Outcome { + let Self { ip, udp, frame, waiting, .. } = self; let mut sent = None; - let served = udp.serve(sender, |out| match ip.send_udp(now, out, frame) { - Ok(Sent::Frame(len)) => { + let room = &*credit; + let served = udp.serve(sender, |out| match ip.send_udp(now, room, out, frame) { + Ok(Sent::Frame { len, iface }) => { sent = frame.get(..len).map(|bytes| { - sink(bytes); - charge(bytes) + sink(iface, bytes); + (iface, charge(bytes)) }); Offer::Taken } Ok(Sent::Pending(next_hop)) => Offer::Waits(next_hop), + Ok(Sent::NoRoom { iface, hop }) => { + waiting.entry(Wait::Link(iface)).or_default().insert(hop); + Offer::Waits(hop) + } // Counted by [ip], which tells the datagram's flow. Err(_) => Offer::Taken, }); - Outcome { frame: sent, leaves: served != UdpServed::More } + if let Some((iface, _)) = sent { + credit.spend(iface); + } + Outcome { frame: sent.map(|(_, len)| len), leaves: served != UdpServed::More } } pub fn next_deadline(&self) -> Option { @@ -408,10 +448,13 @@ impl Shard { // No datagram waits on a failed next hop: the failure dropped them. toyos_net_ip::Event::Cleared { next_hop, .. } => wake(tcp, waiting, Wait::Hop(next_hop)), toyos_net_ip::Event::Room { .. } => wake(tcp, waiting, Wait::Room), - toyos_net_ip::Event::Verified { addr, .. } => events.push(Event::Verified(addr)), - toyos_net_ip::Event::Conflict { addr, mac, .. } => events.push(Event::Conflict { addr, mac }), - toyos_net_ip::Event::Lost { addr, mac, .. } => events.push(Event::Lost { addr, mac }), - toyos_net_ip::Event::NotVerified { addr, .. } => events.push(Event::NotVerified(addr)), + toyos_net_ip::Event::Verified { iface, addr } => events.push(Event::Verified { iface, addr }), + toyos_net_ip::Event::Conflict { iface, addr, mac } => events.push(Event::Conflict { iface, addr, mac }), + toyos_net_ip::Event::Lost { iface, addr, mac } => { + tcp.lose_address(addr); + events.push(Event::Lost { iface, addr, mac }); + } + toyos_net_ip::Event::NotVerified { iface, addr } => events.push(Event::NotVerified { iface, addr }), } } for r in udp.drain_refusals() { @@ -440,44 +483,47 @@ impl Shard { // ---- configuration, the shell's ---- - pub fn link_up(&mut self, now: Instant) -> Result<(), toyos_net_ip::Counter> { - let up = self.ip.link_up(now, self.iface); + pub fn link_up(&mut self, now: Instant, iface: IfIndex) -> Result<(), toyos_net_ip::Counter> { + let up = self.ip.link_up(now, iface); self.settle(now); up } - pub fn link_down(&mut self, now: Instant) -> Result<(), toyos_net_ip::Counter> { - let down = self.ip.link_down(now, self.iface); + pub fn link_down(&mut self, now: Instant, iface: IfIndex) -> Result<(), toyos_net_ip::Counter> { + let down = self.ip.link_down(now, iface); self.settle(now); down } /// Added tentative: conflict detection runs before it is used ([`Event::Verified`]). - pub fn add_address(&mut self, now: Instant, addr: Ipv4Addr, prefix_len: u8) -> Result<(), toyos_net_ip::Counter> { - let added = self.ip.add_address(now, self.iface, addr, prefix_len); + pub fn add_address(&mut self, now: Instant, iface: IfIndex, addr: Ipv4Addr, prefix_len: u8) -> Result<(), toyos_net_ip::Counter> { + let added = self.ip.add_address(now, iface, addr, prefix_len); self.settle(now); added } - pub fn set_gateways(&mut self, now: Instant, gateways: &[Ipv4Addr]) -> Result<(), toyos_net_ip::Counter> { - let set = self.ip.set_gateways(now, self.iface, gateways); + pub fn set_gateways(&mut self, now: Instant, iface: IfIndex, gateways: &[Ipv4Addr]) -> Result<(), toyos_net_ip::Counter> { + let set = self.ip.set_gateways(now, iface, gateways); self.settle(now); set } /// Takes one of the interface's addresses away; [ip] withdraws the gateways no usable prefix - /// holds any more. - pub fn remove_address(&mut self, now: Instant, addr: Ipv4Addr) -> Result<(), toyos_net_ip::Counter> { - let removed = self.ip.remove_address(now, self.iface, addr); + /// holds any more, and every TCP connection from it ends (`Failure::AddressGone`). + pub fn remove_address(&mut self, now: Instant, iface: IfIndex, addr: Ipv4Addr) -> Result<(), toyos_net_ip::Counter> { + let removed = self.ip.remove_address(now, iface, addr); + if removed.is_ok() { + self.tcp.lose_address(addr); + } self.settle(now); removed } - /// Joins `group` on the interface: frames to its link address are taken from here on, and - /// [ip] reports the membership whenever the link can carry the report. Nothing to settle: - /// a join makes no flow eligible and no event. - pub fn join(&mut self, now: Instant, group: MulticastAddr) -> Result<(), toyos_net_ip::Counter> { - self.ip.join(now, self.iface, group) + /// Joins `group` on `iface`: frames to its link address are taken from here on, and [ip] + /// reports the membership whenever the link can carry the report. Nothing to settle: a join + /// makes no flow eligible and no event. + pub fn join(&mut self, now: Instant, iface: IfIndex, group: MulticastAddr) -> Result<(), toyos_net_ip::Counter> { + self.ip.join(now, iface, group) } // ---- TCP ---- @@ -680,10 +726,10 @@ impl Shard { Ok(id) } - /// The acquisition socket's send, naming its source: 0.0.0.0 to the limited broadcast, or an - /// assigned address. - pub fn send_from(&mut self, now: Instant, id: SocketId, source: Ipv4Addr, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), toyos_net_udp::Error> { - let queued = self.udp.send_from(&mut self.ip, id, source, destination, port, payload); + /// The acquisition socket's send, naming its source, 0.0.0.0 to the limited broadcast or an + /// assigned address, and the interface whose address it acquires. + pub fn send_from(&mut self, now: Instant, id: SocketId, origin: Origin, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), toyos_net_udp::Error> { + let queued = self.udp.send_from(&mut self.ip, id, origin, destination, port, payload); self.settle(now); queued } @@ -693,18 +739,24 @@ impl Shard { /// move: only a next hop with no entry is resolved, which queues its request. /// No route is a local destination unreachable (RFC 1122 §3.3.1.1). A flow told to wait is /// recorded under what it waits on; one with no route waits on the routes. -fn hop(ip: &mut Ip, now: Instant, iface: IfIndex, tuple: &Tuple, waiting: &mut BTreeMap>) -> Hop { +fn hop(ip: &mut Ip, now: Instant, credit: &impl Credit, tuple: &Tuple, waiting: &mut BTreeMap>) -> Hop { let remote = tuple.remote.addr; - let Ok(route) = ip.route(remote, Source::Bound(tuple.local.addr), Some(iface)) else { return Hop::Unreachable }; + let Ok(route) = ip.route(remote, Source::Bound(tuple.local.addr), None) else { return Hop::Unreachable }; let NextHop::Neighbour(next_hop) = route.next_hop else { return Hop::Unreachable }; + let iface = route.iface; + let Some(source) = ip.mac(iface) else { unreachable!("a route names an interface [ip] holds") }; + if !credit.room(iface) { + waiting.entry(Wait::Link(iface)).or_default().insert(remote); + return Hop::Pending; + } let (answer, wait) = match ip.neighbour(iface, next_hop) { Some(entry) => match entry.mac() { - Some(mac) => (Hop::Ready(Via { next_hop, mac }), None), + Some(mac) => (Hop::Ready(Via { iface, source, next_hop, mac }), None), None if matches!(entry, Nud::Failed) => (Hop::Unreachable, Some(Wait::Hop(next_hop))), None => (Hop::Pending, Some(Wait::Hop(next_hop))), }, None => match ip.resolve(now, iface, next_hop, route.source) { - Resolution::Resolved(mac) => (Hop::Ready(Via { next_hop, mac }), None), + Resolution::Resolved(mac) => (Hop::Ready(Via { iface, source, next_hop, mac }), None), Resolution::Pending => (Hop::Pending, Some(Wait::Hop(next_hop))), Resolution::Failed => (Hop::Unreachable, Some(Wait::Room)), }, @@ -776,11 +828,12 @@ mod tests { }; let mac = IndividualMac::new(MacAddr([2, 0, 0, 0, 0, 0x0a])).unwrap(); let mut now = Instant::from_millis(3_600_000); - let mut shard = Shard::new(now, Config { mac, receive_buffer: 65_535, send_buffer: 65_535, secrets }).unwrap(); - shard.link_up(now).unwrap(); - shard.add_address(now, A, 24).unwrap(); - while !shard.drain_events().any(|e| e == Event::Verified(A)) { - shard.transmit(now, usize::MAX, |_| {}); + let mut shard = Shard::new(now, Config { receive_buffer: 65_535, send_buffer: 65_535, secrets }).unwrap(); + let iface = shard.add_interface(now, mac, Rank(0)); + shard.link_up(now, iface).unwrap(); + shard.add_address(now, iface, A, 24).unwrap(); + while !shard.drain_events().any(|e| matches!(e, Event::Verified { addr: A, .. })) { + shard.transmit(now, usize::MAX, |_, _| {}); now = shard.next_deadline().expect("conflict detection is still running"); shard.fire(now); } @@ -809,13 +862,13 @@ mod tests { const C: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 3); const D: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 4); let (mut shard, now) = verified(); - shard.transmit(now, usize::MAX, |_| {}); + shard.transmit(now, usize::MAX, |_, _| {}); let port = |n| Port::new(n).unwrap(); let (to_c, to_d) = (shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5000)), || 0).unwrap(), shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5001)), || 0).unwrap()); shard.send_to(now, to_c, C, 9, b"c").unwrap(); shard.send_to(now, to_d, D, 9, b"d").unwrap(); assert_eq!(shard.round.len(), 2); - assert_eq!(shard.transmit(now, usize::MAX, |_| {}), 2, "a request for each"); + assert_eq!(shard.transmit(now, usize::MAX, |_, _| {}), 2, "a request for each"); assert!(shard.round.is_empty(), "what waits is not served"); let mut reply = [[2, 0, 0, 0, 0, 0x0a], [2, 0, 0, 0, 0, 0x0c]].concat(); @@ -823,7 +876,7 @@ mod tests { reply.extend_from_slice(&C.octets()); reply.extend_from_slice(&[2, 0, 0, 0, 0, 0x0a]); reply.extend_from_slice(&A.octets()); - shard.receive(now, &reply); + shard.receive(now, shard.ifaces[0], &reply); assert_eq!(shard.round.iter().map(|m| m.flow).collect::>(), [Flow::Udp(Sender::Socket(to_c))]); } diff --git a/system/netstack/shard/tcp/src/lib.rs b/system/netstack/shard/tcp/src/lib.rs index 05a27c8cb9e..cb131323458 100644 --- a/system/netstack/shard/tcp/src/lib.rs +++ b/system/netstack/shard/tcp/src/lib.rs @@ -212,6 +212,8 @@ pub enum Failure { Unreachable(SoftError), /// ICMP destination unreachable, administratively prohibited, during the handshake. Prohibited, + /// The host gave up the connection's local address ([`Tcp::lose_address`]). + AddressGone, } /// An ICMP error that did not end the connection; reported as the reason if it later gives up. diff --git a/system/netstack/shard/tcp/src/stack.rs b/system/netstack/shard/tcp/src/stack.rs index fe9c0b0f5e6..aef8287d958 100644 --- a/system/netstack/shard/tcp/src/stack.rs +++ b/system/netstack/shard/tcp/src/stack.rs @@ -1112,6 +1112,22 @@ impl Tcp { Ok(()) } + /// The host no longer holds `local`: every connection from it ends with + /// [`Failure::AddressGone`], with no reset, since nothing can be sent from it. A connection + /// cannot move to another address, so waiting would only end it later by its timeout. + pub fn lose_address(&mut self, local: Ipv4Addr) { + let lost: Vec = self + .conns + .iter() + .enumerate() + .filter_map(|(index, slot)| slot.value.as_ref().filter(|c| c.tuple.local.addr == local && !matches!(c.state, Tcb::Ended(_))).map(|_| index)) + .filter_map(|index| u32::try_from(index).ok()) + .collect(); + for index in lost { + self.end(index, Some(Failure::AddressGone), None); + } + } + /// RFC 9293 §3.10.5. pub fn abort(&mut self, now: Instant, id: ConnId) -> Result<(), Error> { let conn = self.conn(id)?; diff --git a/system/netstack/shard/tcp/tests/close.rs b/system/netstack/shard/tcp/tests/close.rs index 75e96652fef..8da10d95866 100644 --- a/system/netstack/shard/tcp/tests/close.rs +++ b/system/netstack/shard/tcp/tests/close.rs @@ -459,3 +459,18 @@ fn s_cl_032_rst_in_last_ack_and_closing() { nothing(&h.input(10, seg(5002).rst())); assert!(!held(&mut h)); } + +/// A connection whose local address the host gave up ends at once, named for +/// it, and sends nothing: no reset can leave from an address it no longer +/// holds. One from another address is untouched. +#[test] +fn a_connection_from_a_lost_address_ends_and_sends_nothing() { + let mut h = fixture_e(); + let other = h.tcp.connect(h.now(), std::net::Ipv4Addr::new(198, 51, 100, 1), Some(port(49153)), ep(B, 80)).unwrap(); + h.transmit(); + h.tcp.lose_address(A); + nothing(&h.transmit()); + assert_eq!(h.call(10, |tcp, now, id| tcp.send(now, id, b"x")).0, Err(Error::Failed(Failure::AddressGone))); + let status = h.tcp.status(other).unwrap(); + assert_eq!(status.state, State::SynSent, "the other address's connection is left"); +} diff --git a/system/netstack/shard/testnet/Cargo.toml b/system/netstack/shard/testnet/Cargo.toml index 6b21b363e4c..86f05c229c3 100644 --- a/system/netstack/shard/testnet/Cargo.toml +++ b/system/netstack/shard/testnet/Cargo.toml @@ -7,6 +7,7 @@ description = "A fake Ethernet for toyos-net-shard's host tests: N shards on one publish = false [dependencies] +toyos-net-ip = { path = "../../ip" } toyos-net-shard = { path = ".." } toyos-net-tcp = { path = "../tcp" } toyos-net-wire = { path = "../../wire" } diff --git a/system/netstack/shard/testnet/src/lib.rs b/system/netstack/shard/testnet/src/lib.rs index e91e3d6a056..f56d6bfc349 100644 --- a/system/netstack/shard/testnet/src/lib.rs +++ b/system/netstack/shard/testnet/src/lib.rs @@ -19,7 +19,7 @@ use std::collections::BTreeMap; use std::net::Ipv4Addr; use std::time::Duration; -use toyos_net_shard::{Config, Event, Secrets, Shard}; +use toyos_net_shard::{Config, Event, IfIndex, Rank, Secrets, Shard}; use toyos_net_tcp::{ConnId, Endpoint, Error, Failure, ListenerId, Received, Tuple}; use toyos_net_wire::ethernet::{Frame, IndividualMac, MacAddr, MacClass}; use toyos_net_wire::{Instant, Port}; @@ -82,6 +82,8 @@ impl Link { pub struct Node { pub shard: Shard, + /// Its one interface, on the segment. + pub iface: IfIndex, pub mac: MacAddr, pub credit: Credit, /// What the shard reported, oldest first. @@ -89,6 +91,33 @@ pub struct Node { budget: (u64, usize), } +impl Node { + pub fn receive(&mut self, now: Instant, frame: &[u8]) { + self.shard.receive(now, self.iface, frame); + } + + pub fn link_up(&mut self, now: Instant) -> Result<(), toyos_net_ip::Counter> { + self.shard.link_up(now, self.iface) + } + + pub fn link_down(&mut self, now: Instant) -> Result<(), toyos_net_ip::Counter> { + self.shard.link_down(now, self.iface) + } + + pub fn add_address(&mut self, now: Instant, addr: Ipv4Addr, prefix_len: u8) -> Result<(), toyos_net_ip::Counter> { + self.shard.add_address(now, self.iface, addr, prefix_len) + } + + pub fn set_gateways(&mut self, now: Instant, gateways: &[Ipv4Addr]) -> Result<(), toyos_net_ip::Counter> { + self.shard.set_gateways(now, self.iface, gateways) + } + + /// Whether the shard has reported `addr` verified. + pub fn verified(&self, addr: Ipv4Addr) -> bool { + self.events.iter().any(|e| matches!(e, Event::Verified { addr: a, .. } if *a == addr)) + } +} + /// A frame as a node's device put it on the wire. #[derive(Clone, Debug)] pub struct Carried { @@ -245,15 +274,16 @@ impl Net { port_table: [0; 16], }, }; - let config = Config { mac: IndividualMac::new(mac).expect("an individual MAC"), receive_buffer: 65_535, send_buffer: 65_535, secrets }; + let config = Config { receive_buffer: 65_535, send_buffer: 65_535, secrets }; let mut shard = Shard::new(self.now, config).expect("a valid configuration"); - shard.link_up(self.now).expect("its own interface"); - shard.add_address(self.now, addr, prefix_len).expect("a valid address"); + let iface = shard.add_interface(self.now, IndividualMac::new(mac).expect("an individual MAC"), Rank(0)); + shard.link_up(self.now, iface).expect("its own interface"); + shard.add_address(self.now, iface, addr, prefix_len).expect("a valid address"); for other in 0..index { self.links.insert((index, other), Link::default()); self.links.insert((other, index), Link::default()); } - self.nodes.push(Node { shard, mac, credit: Credit::Unlimited, events: Vec::new(), budget: (u64::MAX, 0) }); + self.nodes.push(Node { shard, iface, mac, credit: Credit::Unlimited, events: Vec::new(), budget: (u64::MAX, 0) }); index } @@ -319,7 +349,8 @@ impl Net { let due = self.due(); let mut busy = !due.is_empty(); for flight in due { - self.nodes[flight.to].shard.receive(self.now, &flight.frame); + let node = &mut self.nodes[flight.to]; + node.shard.receive(self.now, node.iface, &flight.frame); } for node in &mut self.nodes { node.shard.fire(self.now); @@ -363,7 +394,8 @@ impl Net { } }; let mut frames = Vec::new(); - let spent = node.shard.transmit(now, credit, |frame| frames.push(frame.to_vec())); + let mut credit = credit; + let spent = node.shard.transmit(now, &mut credit, |_, frame| frames.push(frame.to_vec())); if let Credit::PerMs(_) = node.credit { node.budget.1 -= spent; } diff --git a/system/netstack/shard/tests/acquisition.rs b/system/netstack/shard/tests/acquisition.rs index 2b0258dd788..2e53bc46101 100644 --- a/system/netstack/shard/tests/acquisition.rs +++ b/system/netstack/shard/tests/acquisition.rs @@ -7,7 +7,7 @@ mod common; use std::net::Ipv4Addr; use common::*; -use toyos_net_shard::{Config, Event, Secrets, Shard}; +use toyos_net_shard::{Config, Event, IfIndex, Origin, Rank, Secrets, Shard}; use toyos_net_udp::{Counter, Error}; use toyos_net_wire::arp::Arp; use toyos_net_wire::ethernet::{EtherType, Frame, IndividualMac, MacAddr}; @@ -16,7 +16,7 @@ use toyos_net_wire::udp::UdpDatagram; use toyos_net_wire::Instant; /// A's shard an hour into the clock, its link up and no address on it. -fn unaddressed() -> (Shard, Instant) { +fn unaddressed() -> (Shard, IfIndex, Instant) { let secrets = Secrets { ip: [1; 16], resets: [2; 16], @@ -24,21 +24,22 @@ fn unaddressed() -> (Shard, Instant) { }; let now = Instant::from_millis(3_600_000); let mac = IndividualMac::new(MAC_A).unwrap(); - let mut shard = Shard::new(now, Config { mac, receive_buffer: 65_535, send_buffer: 65_535, secrets }).unwrap(); - shard.link_up(now).unwrap(); - (shard, now) + let mut shard = Shard::new(now, Config { receive_buffer: 65_535, send_buffer: 65_535, secrets }).unwrap(); + let iface = shard.add_interface(now, mac, Rank(0)); + shard.link_up(now, iface).unwrap(); + (shard, iface, now) } fn sent(shard: &mut Shard, now: Instant) -> Vec> { let mut frames = Vec::new(); - shard.transmit(now, usize::MAX, |frame| frames.push(frame.to_vec())); + shard.transmit(now, usize::MAX, |_, frame| frames.push(frame.to_vec())); frames } /// A's address verified, and the time it was. -fn verified(shard: &mut Shard, mut now: Instant) -> Instant { - shard.add_address(now, A, 24).unwrap(); - while !shard.drain_events().any(|e| e == Event::Verified(A)) { +fn verified(shard: &mut Shard, iface: IfIndex, mut now: Instant) -> Instant { + shard.add_address(now, iface, A, 24).unwrap(); + while !shard.drain_events().any(|e| matches!(e, Event::Verified { addr: A, .. })) { sent(shard, now); now = shard.next_deadline().expect("conflict detection is still running"); shard.fire(now); @@ -80,9 +81,9 @@ fn who_has(target: Ipv4Addr) -> Vec { #[test] fn the_acquisition_socket_sends_from_no_address_to_the_limited_broadcast() { - let (mut shard, now) = unaddressed(); + let (mut shard, iface, now) = unaddressed(); let socket = shard.acquisition().unwrap(); - shard.send_from(now, socket, Ipv4Addr::UNSPECIFIED, Ipv4Addr::BROADCAST, 67, b"discover").unwrap(); + shard.send_from(now, socket, Origin { iface, source: Ipv4Addr::UNSPECIFIED }, Ipv4Addr::BROADCAST, 67, b"discover").unwrap(); let frames = sent(&mut shard, now); let [frame] = &frames[..] else { panic!("one frame, not {}", frames.len()) }; let frame = Frame::parse(frame).unwrap(); @@ -95,9 +96,9 @@ fn the_acquisition_socket_sends_from_no_address_to_the_limited_broadcast() { #[test] fn the_acquisition_socket_hears_a_datagram_to_an_address_not_yet_held() { - let (mut shard, now) = unaddressed(); + let (mut shard, iface, now) = unaddressed(); let socket = shard.acquisition().unwrap(); - shard.receive(now, &from_router(MAC_A, A, b"offer")); + shard.receive(now, iface, &from_router(MAC_A, A, b"offer")); let mut buf = [0u8; 16]; let received = shard.recv_from(socket, &mut buf).unwrap().expect("the acquisition exception admits it"); assert_eq!((&buf[..received.len], received.source, received.source_port), (&b"offer"[..], R, Some(port(67)))); @@ -105,30 +106,30 @@ fn the_acquisition_socket_hears_a_datagram_to_an_address_not_yet_held() { #[test] fn only_the_acquisition_socket_names_its_source() { - let (mut shard, now) = unaddressed(); - let now = verified(&mut shard, now); + let (mut shard, iface, now) = unaddressed(); + let now = verified(&mut shard, iface, now); sent(&mut shard, now); let plain = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5000)), || 0).unwrap(); let acquisition = shard.acquisition().unwrap(); // A source the acquisition socket may name: an assigned address, to one host. - assert_eq!(shard.send_from(now, plain, A, R, 67, b"request"), Err(Error::Refused(Counter::SourceNotPermitted))); - assert_eq!(shard.send_from(now, acquisition, A, R, 67, b"request"), Ok(())); + assert_eq!(shard.send_from(now, plain, Origin { iface, source: A }, R, 67, b"request"), Err(Error::Refused(Counter::SourceNotPermitted))); + assert_eq!(shard.send_from(now, acquisition, Origin { iface, source: A }, R, 67, b"request"), Ok(())); } #[test] fn a_second_acquisition_socket_is_refused_its_port() { - let (mut shard, _) = unaddressed(); + let (mut shard, _, _) = unaddressed(); shard.acquisition().unwrap(); assert_eq!(shard.acquisition(), Err(Error::Refused(Counter::PortInUse))); } #[test] fn a_removed_address_takes_its_gateway_and_answers_nothing() { - let (mut shard, now) = unaddressed(); - let now = verified(&mut shard, now); - shard.set_gateways(now, &[R]).unwrap(); + let (mut shard, iface, now) = unaddressed(); + let now = verified(&mut shard, iface, now); + shard.set_gateways(now, iface, &[R]).unwrap(); sent(&mut shard, now); - shard.receive(now, &who_has(A)); + shard.receive(now, iface, &who_has(A)); let answered = sent(&mut shard, now); let replies = |frames: &[Vec]| { let arp = |f: &Vec| Frame::parse(f).ok().filter(|f| f.ether_type() == EtherType::Arp).and_then(|f| Arp::parse(f.body()).ok()); @@ -136,10 +137,10 @@ fn a_removed_address_takes_its_gateway_and_answers_nothing() { }; assert_eq!(replies(&answered), 1, "held, the address answers"); - shard.remove_address(now, A).unwrap(); - assert_eq!(shard.ip().address(shard.iface(), A), None); - assert_eq!(shard.ip().gateways(shard.iface()), Some(&[] as &[Ipv4Addr])); - shard.receive(now, &who_has(A)); + shard.remove_address(now, iface, A).unwrap(); + assert_eq!(shard.ip().address(iface, A), None); + assert_eq!(shard.ip().gateways(iface), Some(&[] as &[Ipv4Addr])); + shard.receive(now, iface, &who_has(A)); assert_eq!(replies(&sent(&mut shard, now)), 0, "removed, it answers nothing"); - assert_eq!(shard.remove_address(now, A), Err(toyos_net_ip::Counter::AddrUnknown)); + assert_eq!(shard.remove_address(now, iface, A), Err(toyos_net_ip::Counter::AddrUnknown)); } diff --git a/system/netstack/shard/tests/common/mod.rs b/system/netstack/shard/tests/common/mod.rs index f54d5823daf..357ad019f9c 100644 --- a/system/netstack/shard/tests/common/mod.rs +++ b/system/netstack/shard/tests/common/mod.rs @@ -8,7 +8,7 @@ use std::net::Ipv4Addr; use std::time::Duration; -use toyos_net_shard::Event; + use toyos_net_tcp::{ConnId, Endpoint, State}; use toyos_net_testnet::Net; use toyos_net_wire::arp::{Arp, Operation}; @@ -39,10 +39,10 @@ pub fn segment() -> (Net, usize, usize) { let mut net = Net::new(Instant::from_millis(3_600_000)); let a = net.add_node(MAC_A, A, 24); let b = net.add_node(MAC_B, B, 24); - let verified = |net: &mut Net, n: usize, addr| net.nodes[n].events.contains(&Event::Verified(addr)); + let verified = |net: &mut Net, n: usize, addr| net.nodes[n].verified(addr); assert!(net.run_until(Duration::from_secs(1), |net| verified(net, a, A) && verified(net, b, B)), "both addresses verified"); net.advance(toyos_net_ip::limits::acd::ANNOUNCE_INTERVAL * 2); - let none = |n: usize, addr| net.nodes[n].shard.ip().neighbour(net.nodes[n].shard.iface(), addr).is_none(); + let none = |n: usize, addr| net.nodes[n].shard.ip().neighbour(net.nodes[n].iface, addr).is_none(); assert!(none(a, B) && none(b, A), "no neighbour entries yet"); (net, a, b) } diff --git a/system/netstack/shard/tests/drr.rs b/system/netstack/shard/tests/drr.rs index 3a7f8fc60a7..3d271736187 100644 --- a/system/netstack/shard/tests/drr.rs +++ b/system/netstack/shard/tests/drr.rs @@ -10,7 +10,7 @@ use std::net::Ipv4Addr; use std::time::Duration; use common::*; -use toyos_net_shard::Event; + use toyos_net_tcp::{ConnId, State}; use toyos_net_testnet::{Credit, Net}; use toyos_net_wire::ethernet::MacAddr; @@ -39,7 +39,7 @@ fn fixture() -> Fixture { let mut net = Net::new(Instant::from_millis(3_600_000)); let a = net.add_node(MAC_A, A, 24); let peers = [(net.add_node(MAC_B, B, 24), B), (net.add_node(MAC_C, C, 24), C), (net.add_node(MAC_D, D, 24), D)]; - let verified = |net: &mut Net| [(a, A)].into_iter().chain(peers).all(|(n, addr)| net.nodes[n].events.contains(&Event::Verified(addr))); + let verified = |net: &mut Net| [(a, A)].into_iter().chain(peers).all(|(n, addr)| net.nodes[n].verified(addr)); assert!(net.run_until(Duration::from_secs(1), verified), "every address verified"); net.advance(toyos_net_ip::limits::acd::ANNOUNCE_INTERVAL * 2); for (n, addr) in peers { @@ -57,7 +57,7 @@ fn fixture() -> Fixture { fn opportunity(f: &mut Fixture, credit: usize) -> Vec<(String, usize)> { let now = f.net.now(); let mut out = Vec::new(); - f.net.nodes[f.a].shard.transmit(now, credit, |frame| out.push((name(frame), frame.len()))); + f.net.nodes[f.a].shard.transmit(now, credit, |_, frame| out.push((name(frame), frame.len()))); out } @@ -164,8 +164,8 @@ fn s_shard_drr_003_ip_and_owed_resets_go_first_and_the_round_resumes() { let (mut f, first) = bulk(); assert_eq!(names(&first), [B_BULK, C_BULK, D_BULK, B_BULK]); let now = f.net.now(); - f.net.nodes[f.a].shard.receive(now, &echo_request()); - f.net.nodes[f.a].shard.receive(now, &arp_request_from_d()); + f.net.nodes[f.a].receive(now, &echo_request()); + f.net.nodes[f.a].receive(now, &arp_request_from_d()); f.net.nodes[f.a].shard.abort(now, f.conns[4]).unwrap(); let out = opportunity(&mut f, 6); assert_eq!( @@ -187,7 +187,7 @@ fn a_connection_that_takes_a_freed_slot_takes_one_turn() { let third = f.net.nodes[a].shard.connect(now, Some(port(49158)), ep(C, 81)).unwrap(); let syn = opportunity_frames(&mut f, 1); assert_eq!(syn.len(), 1); - f.net.nodes[c].shard.receive(now, &syn[0]); + f.net.nodes[c].receive(now, &syn[0]); let established = |net: &mut Net| net.nodes[a].shard.status(third).unwrap().state == State::Established; assert!(f.net.run_until(Duration::from_secs(1), established), "the SYN-ACK arrives"); let now = f.net.now(); @@ -200,7 +200,7 @@ fn a_connection_that_takes_a_freed_slot_takes_one_turn() { fn opportunity_frames(f: &mut Fixture, credit: usize) -> Vec> { let now = f.net.now(); let mut out = Vec::new(); - f.net.nodes[f.a].shard.transmit(now, credit, |frame| out.push(frame.to_vec())); + f.net.nodes[f.a].shard.transmit(now, credit, |_, frame| out.push(frame.to_vec())); out } diff --git a/system/netstack/shard/tests/egress.rs b/system/netstack/shard/tests/egress.rs index 7b07e22cccf..27c6cea50be 100644 --- a/system/netstack/shard/tests/egress.rs +++ b/system/netstack/shard/tests/egress.rs @@ -25,7 +25,7 @@ const MAC_D: MacAddr = MacAddr([0x02, 0, 0, 0, 0, 0x0d]); fn opportunity(net: &mut Net, node: usize, credit: usize) -> Vec { let now = net.now(); let mut out = Vec::new(); - net.nodes[node].shard.transmit(now, credit, |frame| out.push(name(frame))); + net.nodes[node].shard.transmit(now, credit, |_, frame| out.push(name(frame))); out } @@ -93,7 +93,7 @@ fn s_pl_013_shard_a_connect_with_no_route_fails_at_once() { let (mut net, a, _) = segment(); let now = net.now(); let id = net.nodes[a].shard.connect(now, Some(port(49152)), ep(B, 80)).unwrap(); - net.nodes[a].shard.link_down(now).unwrap(); + net.nodes[a].link_down(now).unwrap(); net.advance(Duration::from_secs(5)); assert_eq!(net.nodes[a].shard.status(id).unwrap().failure, Some(HOST_UNREACHABLE)); let counters = net.nodes[a].shard.tcp_counters(); @@ -109,9 +109,9 @@ fn routes_refused(net: &Net, node: usize) -> u64 { /// A forgets every neighbour: a link down and up again. fn forget(net: &mut Net, node: usize) { let now = net.now(); - net.nodes[node].shard.link_down(now).unwrap(); - net.nodes[node].shard.link_up(now).unwrap(); - let iface = net.nodes[node].shard.iface(); + net.nodes[node].link_down(now).unwrap(); + net.nodes[node].link_up(now).unwrap(); + let iface = net.nodes[node].iface; assert!(net.nodes[node].shard.ip().neighbour(iface, B).is_none()); } @@ -178,7 +178,7 @@ fn s_pl_014_shard_an_established_flow_with_no_route_records_host_unreachable() { let (mut net, a, b) = segment(); let (to_b, from_a) = established(&mut net, a, b); let now = net.now(); - net.nodes[a].shard.link_down(now).unwrap(); + net.nodes[a].link_down(now).unwrap(); net.nodes[a].shard.send(now, to_b, &[0x55; 100]).unwrap(); net.advance(Duration::from_secs(10)); let status = net.nodes[a].shard.status(to_b).unwrap(); @@ -187,7 +187,7 @@ fn s_pl_014_shard_an_established_flow_with_no_route_records_host_unreachable() { // A route again: the flow asks, and its segment leaves; B's acknowledgment clears the error. let now = net.now(); - net.nodes[a].shard.link_up(now).unwrap(); + net.nodes[a].link_up(now).unwrap(); net.advance(Duration::from_secs(1)); assert_eq!(net.nodes[a].shard.status(to_b).unwrap().soft_error, None); let now = net.now(); @@ -200,7 +200,7 @@ fn s_pl_014_shard_an_established_flow_with_no_route_records_host_unreachable() { fn b_failed_d_ready() -> (Net, usize, toyos_net_tcp::ConnId, toyos_net_tcp::ConnId) { let (mut net, a, b) = segment(); let d = net.add_node(MAC_D, D, 24); - assert!(net.run_until(Duration::from_secs(1), |net| net.nodes[d].events.contains(&toyos_net_shard::Event::Verified(D)))); + assert!(net.run_until(Duration::from_secs(1), |net| net.nodes[d].verified(D))); net.advance(toyos_net_ip::limits::acd::ANNOUNCE_INTERVAL * 2); let (to_b, _) = established(&mut net, a, b); let (to_d, _) = connected(&mut net, a, 49153, d, D); @@ -211,7 +211,7 @@ fn b_failed_d_ready() -> (Net, usize, toyos_net_tcp::ConnId, toyos_net_tcp::Conn net.nodes[a].shard.send_to(now, socket, B, 9, b"x").unwrap(); net.nodes[a].shard.send_to(now, socket, D, 9, b"x").unwrap(); net.advance(Duration::from_secs(4)); - let iface = net.nodes[a].shard.iface(); + let iface = net.nodes[a].iface; assert!(matches!(net.nodes[a].shard.ip().neighbour(iface, B), Some(Nud::Failed))); assert!(net.nodes[a].shard.ip().neighbour(iface, D).is_some_and(|n| n.mac().is_some())); (net, a, to_b, to_d) @@ -237,7 +237,7 @@ fn s_pl_015_shard_a_reset_with_no_route_is_dropped() { let (mut net, a, b) = segment(); let (to_b, _) = established(&mut net, a, b); let now = net.now(); - net.nodes[a].shard.link_down(now).unwrap(); + net.nodes[a].link_down(now).unwrap(); let start = net.wire().len(); net.nodes[a].shard.abort(now, to_b).unwrap(); net.advance(Duration::from_secs(5)); @@ -252,7 +252,7 @@ fn s_hs_034_shard_a_child_whose_peer_never_answers_gives_up() { let start = net.wire().len(); let now = net.now(); net.link(b, a).dark = Some((now, now.after(Duration::from_secs(3_600)))); - net.nodes[a].shard.receive(now, &from_b(&segment_bytes((B, 40_000), (A, 80), 5_000, 0x02))); + net.nodes[a].receive(now, &from_b(&segment_bytes((B, 40_000), (A, 80), 5_000, 0x02))); let given_up = |net: &Net| net.nodes[a].shard.tcp_counters().get(Counter::SynAckGiveUp); net.advance(Duration::from_millis(59_999)); assert_eq!(given_up(&net), 0); @@ -327,19 +327,19 @@ fn stale_peer() -> (Net, usize, toyos_net_tcp::ConnId) { let (to_b, _) = established(&mut net, a, b); forget(&mut net, a); let now = net.now(); - net.nodes[b].shard.link_down(now).unwrap(); - net.nodes[b].shard.link_up(now).unwrap(); + net.nodes[b].link_down(now).unwrap(); + net.nodes[b].link_up(now).unwrap(); net.nodes[a].shard.bind(A, Some(port(9)), || 0).unwrap(); let socket = net.nodes[b].shard.bind(B, None, || 0).unwrap(); net.nodes[b].shard.send_to(now, socket, A, 9, b"x").unwrap(); net.advance(Duration::from_millis(1)); - let iface = net.nodes[a].shard.iface(); + let iface = net.nodes[a].iface; assert!(matches!(net.nodes[a].shard.ip().neighbour(iface, B), Some(Nud::Stale(_)))); (net, a, to_b) } fn state_of_b(net: &Net, a: usize) -> Option<&Nud> { - net.nodes[a].shard.ip().neighbour(net.nodes[a].shard.iface(), B) + net.nodes[a].shard.ip().neighbour(net.nodes[a].iface, B) } #[test] @@ -367,10 +367,10 @@ fn s_ip_nud_009_shard_an_acknowledgment_ends_delay() { fn s_ip_nud_020_shard_a_stalled_connect_reverifies_the_gateway() { let (mut net, a, _) = segment(); let r = net.add_node(MAC_R, R, 24); - assert!(net.run_until(Duration::from_secs(1), |net| net.nodes[r].events.contains(&toyos_net_shard::Event::Verified(R)))); + assert!(net.run_until(Duration::from_secs(1), |net| net.nodes[r].verified(R))); net.advance(toyos_net_ip::limits::acd::ANNOUNCE_INTERVAL * 2); let now = net.now(); - net.nodes[a].shard.set_gateways(now, &[R]).unwrap(); + net.nodes[a].set_gateways(now, &[R]).unwrap(); let start = net.wire().len(); net.nodes[a].shard.connect(now, Some(port(49152)), ep(std::net::Ipv4Addr::new(198, 51, 100, 7), 80)).unwrap(); assert!(net.run_until(Duration::from_secs(1), |net| sent(net, a, start).iter().any(|f| f.contains("SYN")))); @@ -380,7 +380,7 @@ fn s_ip_nud_020_shard_a_stalled_connect_reverifies_the_gateway() { assert!(!sent(&net, a, start).contains(&"ARP poll 192.0.2.254".to_string()), "{}", dump(&net, start)); net.advance(Duration::from_millis(1)); assert!(sent(&net, a, start).contains(&"ARP poll 192.0.2.254".to_string()), "{}", dump(&net, start)); - let iface = net.nodes[a].shard.iface(); + let iface = net.nodes[a].iface; assert!(net.nodes[a].shard.ip().neighbour(iface, std::net::Ipv4Addr::new(198, 51, 100, 7)).is_none()); } @@ -389,7 +389,7 @@ fn s_ip_nud_022_shard_a_flow_refused_by_a_full_table_asks_again_once_an_entry_ma let mut net = Net::new(Instant::from_millis(3_600_000)); let a = net.add_node(MAC_A, A, 22); let b = net.add_node(MAC_B, B, 22); - let verified = |net: &mut Net, n: usize, addr| net.nodes[n].events.contains(&toyos_net_shard::Event::Verified(addr)); + let verified = |net: &mut Net, n: usize, addr| net.nodes[n].verified(addr); assert!(net.run_until(Duration::from_secs(1), |net| verified(net, a, A) && verified(net, b, B))); net.advance(toyos_net_ip::limits::acd::ANNOUNCE_INTERVAL * 2); let (to_b, from_a) = established(&mut net, a, b); @@ -426,14 +426,14 @@ fn s_ip_nud_024_shard_a_segments_request_prefers_its_source() { let (mut net, a, _) = segment(); let (second, peer) = (Ipv4Addr::new(198, 51, 100, 1), Ipv4Addr::new(198, 51, 100, 9)); let now = net.now(); - net.nodes[a].shard.add_address(now, second, 24).unwrap(); - assert!(net.run_until(Duration::from_secs(1), |net| net.nodes[a].events.contains(&toyos_net_shard::Event::Verified(second)))); + net.nodes[a].add_address(now, second, 24).unwrap(); + assert!(net.run_until(Duration::from_secs(1), |net| net.nodes[a].verified(second))); net.nodes[a].shard.listen(A, Some(port(80)), || 0).unwrap(); let start = net.wire().len(); let mut syn = from_b(&segment_bytes((peer, 40_000), (A, 80), 5_000, 0x02)); syn[6..12].copy_from_slice(&[0x02, 0, 0, 0, 0, 0x99]); let now = net.now(); - net.nodes[a].shard.receive(now, &syn); + net.nodes[a].receive(now, &syn); net.advance(Duration::from_millis(1)); let request = net.wire()[start..].iter().find(|c| c.from == a && name(&c.frame) == format!("ARP request {peer}")).expect("a request"); let frame = toyos_net_wire::ethernet::Frame::parse(&request.frame).unwrap(); diff --git a/system/netstack/shard/tests/icmp.rs b/system/netstack/shard/tests/icmp.rs index ec3dc3fe50d..2f16afb1d76 100644 --- a/system/netstack/shard/tests/icmp.rs +++ b/system/netstack/shard/tests/icmp.rs @@ -33,7 +33,7 @@ fn outstanding() -> (Net, usize, ConnId, Vec) { fn inject(net: &mut Net, node: usize, frame: &[u8]) { let now = net.now(); - net.nodes[node].shard.receive(now, frame); + net.nodes[node].receive(now, frame); } #[test] diff --git a/system/netstack/shard/tests/links.rs b/system/netstack/shard/tests/links.rs new file mode 100644 index 00000000000..d9f472ee47e --- /dev/null +++ b/system/netstack/shard/tests/links.rs @@ -0,0 +1,170 @@ +//! One shard over two links: a new flow leaves by the link that ranks first and has a route, +//! fails over and returns with it, waits for its own link's room without holding back the +//! other's, and ends when its address is lost. + +use std::net::Ipv4Addr; + +use toyos_net_shard::{Config, Credit, Event, IfIndex, Rank, Secrets, Shard}; +use toyos_net_tcp::{Endpoint, Error, Failure}; +use toyos_net_wire::arp::{Arp, Operation}; +use toyos_net_wire::ethernet::{EtherType, Frame, FrameBuilder, IndividualMac, MacAddr}; +use toyos_net_wire::ipv4::Ipv4Packet; +use toyos_net_wire::{Instant, Port}; + +const WIRED_MAC: MacAddr = MacAddr([2, 0, 0, 0, 0, 0x01]); +const WIRELESS_MAC: MacAddr = MacAddr([2, 0, 0, 0, 0, 0x02]); +const WIRED: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 10); +const WIRED_ROUTER: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 1); +const WIRELESS: Ipv4Addr = Ipv4Addr::new(198, 51, 100, 10); +const WIRELESS_ROUTER: Ipv4Addr = Ipv4Addr::new(198, 51, 100, 1); +const ROUTER_MAC: MacAddr = MacAddr([2, 0, 0, 0, 0, 0xfe]); +/// Somewhere only a router reaches. +const FAR: Ipv4Addr = Ipv4Addr::new(203, 0, 113, 7); + +struct Machine { + shard: Shard, + wired: IfIndex, + wireless: IfIndex, + now: Instant, +} + +fn secrets() -> Secrets { + Secrets { + ip: [1; 16], + resets: [2; 16], + tcp: toyos_net_tcp::Secrets { isn: [3; 16], timestamp: [4; 16], port_offset: [5; 16], port_index: [6; 16], port_table: [0; 16] }, + } +} + +/// An ARP reply from a router at `ip` to `to`. +fn router_reply(ip: Ipv4Addr, to: (MacAddr, Ipv4Addr)) -> Vec { + let arp = Arp { operation: Operation::Reply, sender_mac: ROUTER_MAC, sender_ip: ip, target_mac: to.0, target_ip: to.1 }; + let mut frame = vec![0; 64]; + let len = FrameBuilder { destination: to.0, source: IndividualMac::new(ROUTER_MAC).unwrap() }.emit(&arp, &mut frame).unwrap().len(); + frame.truncate(len); + frame +} + +/// The wireless link added first and the wired one second, both up, each address verified +/// and its router known. +fn machine() -> Machine { + let mut now = Instant::from_millis(3_600_000); + let mut shard = Shard::new(now, Config { receive_buffer: 65_535, send_buffer: 65_535, secrets: secrets() }).unwrap(); + let wireless = shard.add_interface(now, IndividualMac::new(WIRELESS_MAC).unwrap(), Rank(1)); + let wired = shard.add_interface(now, IndividualMac::new(WIRED_MAC).unwrap(), Rank(0)); + for (iface, addr) in [(wireless, WIRELESS), (wired, WIRED)] { + shard.link_up(now, iface).unwrap(); + shard.add_address(now, iface, addr, 24).unwrap(); + } + let mut verified = 0; + while verified < 2 { + shard.transmit(now, usize::MAX, |_, _| {}); + verified += shard.drain_events().filter(|e| matches!(e, Event::Verified { .. })).count(); + if verified < 2 { + now = shard.next_deadline().expect("conflict detection is still running"); + shard.fire(now); + } + } + shard.set_gateways(now, wireless, &[WIRELESS_ROUTER]).unwrap(); + shard.set_gateways(now, wired, &[WIRED_ROUTER]).unwrap(); + Machine { shard, wired, wireless, now } +} + +fn far(port: u16) -> Endpoint { + Endpoint { addr: FAR, port: Port::new(port).unwrap() } +} + +/// The IPv4 frames that left with `credit`, each by its interface and source, a router +/// answering every ARP request for it as it leaves, until nothing more leaves. +fn left(m: &mut Machine, mut credit: impl Credit) -> Vec<(IfIndex, Ipv4Addr)> { + let mut out = Vec::new(); + loop { + let mut asked = Vec::new(); + let sent = m.shard.transmit(m.now, &mut credit, |iface, bytes| { + let frame = Frame::parse(bytes).unwrap(); + match frame.ether_type() { + EtherType::Ipv4 => out.push((iface, Ipv4Packet::parse(frame.body()).unwrap().source())), + EtherType::Arp => { + let arp = Arp::parse(frame.body()).unwrap(); + if arp.operation == Operation::Request && [WIRED_ROUTER, WIRELESS_ROUTER].contains(&arp.target_ip) { + asked.push((iface, router_reply(arp.target_ip, (arp.sender_mac, arp.sender_ip)))); + } + } + EtherType::Other(_) => {} + } + }); + for (iface, reply) in &asked { + m.shard.receive(m.now, *iface, reply); + } + if sent == 0 && asked.is_empty() { + return out; + } + } +} + +#[test] +fn a_new_flow_leaves_by_the_wired_link_though_the_wireless_came_first() { + let mut m = machine(); + m.shard.connect(m.now, None, far(80)).unwrap(); + let (wired, out) = (m.wired, left(&mut m, usize::MAX)); + assert_eq!(out, [(wired, WIRED)], "its SYN"); +} + +#[test] +fn a_new_flow_fails_over_to_the_wireless_link_and_the_next_returns_to_the_wired_one() { + let mut m = machine(); + m.shard.link_down(m.now, m.wired).unwrap(); + m.shard.connect(m.now, None, far(80)).unwrap(); + let (wireless, wired) = (m.wireless, m.wired); + assert_eq!(left(&mut m, usize::MAX), [(wireless, WIRELESS)], "failed over"); + m.shard.link_up(m.now, wired).unwrap(); + let back = left(&mut m, usize::MAX); + assert!(back.is_empty(), "nothing moves when the wired link returns: {back:?}"); + m.shard.connect(m.now, None, far(81)).unwrap(); + assert_eq!(left(&mut m, usize::MAX), [(wired, WIRED)], "the next flow is the wired link's again"); +} + +/// Room on one interface alone. +struct Only(IfIndex); + +impl Credit for Only { + fn room(&self, iface: IfIndex) -> bool { + iface == self.0 + } + fn spend(&mut self, _: IfIndex) {} +} + +#[test] +fn a_flow_waits_for_its_own_links_room_and_holds_back_none_of_the_others() { + let mut m = machine(); + let (wired, wireless) = (m.wired, m.wireless); + m.shard.connect(m.now, None, far(80)).unwrap(); + m.shard.link_down(m.now, wired).unwrap(); + m.shard.connect(m.now, None, far(81)).unwrap(); + m.shard.link_up(m.now, wired).unwrap(); + // The wired flow's SYN was due first; with room on the wireless link alone, the wireless + // flow's leaves and the wired one's waits. + assert_eq!(left(&mut m, Only(wireless)), [(wireless, WIRELESS)]); + assert_eq!(left(&mut m, Only(wired)), [(wired, WIRED)], "the wired flow, at the wired link's next room"); +} + +#[test] +fn a_flow_from_an_address_taken_away_ends_and_one_from_the_other_link_does_not() { + let mut m = machine(); + let wired_flow = m.shard.connect(m.now, None, far(80)).unwrap(); + m.shard.link_down(m.now, m.wired).unwrap(); + let wireless_flow = m.shard.connect(m.now, None, far(81)).unwrap(); + m.shard.link_up(m.now, m.wired).unwrap(); + assert_eq!(left(&mut m, usize::MAX).len(), 2, "both SYNs"); + m.shard.remove_address(m.now, m.wired, WIRED).unwrap(); + assert_eq!(m.shard.send(m.now, wired_flow, b"x"), Err(Error::Failed(Failure::AddressGone))); + assert!(m.shard.status(wireless_flow).is_ok_and(|s| s.state == toyos_net_tcp::State::SynSent)); +} + +/// A link going down takes no address, so it ends no flow as its address's loss does. +#[test] +fn a_link_that_only_goes_down_keeps_its_address() { + let mut m = machine(); + m.shard.link_down(m.now, m.wired).unwrap(); + assert!(m.shard.ip().address(m.wired, WIRED).is_some()); +} diff --git a/system/netstack/shard/tests/log.rs b/system/netstack/shard/tests/log.rs index 64dc65348d2..fa426d01ddc 100644 --- a/system/netstack/shard/tests/log.rs +++ b/system/netstack/shard/tests/log.rs @@ -30,13 +30,13 @@ fn refusals(events: &[Event]) -> Vec<(Refusal, u64)> { fn s_ip_mod_001_shard_one_line_per_rule_per_10_s() { let (mut net, a, _) = segment(); let start = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; shard.drain_events().for_each(drop); let frame = from_b(&hex(V_IP_LSRR)); for n in 0..100 { - shard.receive(start.after(Duration::from_millis(n * 10)), &frame); + shard.receive(start.after(Duration::from_millis(n * 10)), iface, &frame); } - shard.receive(start.after(Duration::from_millis(10_001)), &frame); + shard.receive(start.after(Duration::from_millis(10_001)), iface, &frame); let lines = refusals(&shard.drain_events().collect::>()); let rules: Vec<(toyos_net_ip::Counter, Peer, u64)> = lines .iter() @@ -54,13 +54,13 @@ fn s_ip_mod_001_shard_one_line_per_rule_per_10_s() { fn s_mod_005_shard_one_line_per_rule_per_10_s() { let (mut net, a, _) = segment(); let start = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; shard.drain_events().for_each(drop); let syn_fin = |n: u8| from_b(&segment_bytes((Ipv4Addr::new(192, 0, 2, 10 + n), 1234), (A, 80), 5_000, 0x03)); for n in 0..50 { - shard.receive(start.after(Duration::from_millis(u64::from(n))), &syn_fin(n)); + shard.receive(start.after(Duration::from_millis(u64::from(n))), iface, &syn_fin(n)); } - shard.receive(start.after(Duration::from_millis(10_000)), &syn_fin(50)); + shard.receive(start.after(Duration::from_millis(10_000)), iface, &syn_fin(50)); let lines: Vec<(Ipv4Addr, u16, u64)> = refusals(&shard.drain_events().collect::>()) .into_iter() .map(|(r, n)| match r { diff --git a/system/netstack/shard/tests/udp.rs b/system/netstack/shard/tests/udp.rs index 846a2aa746e..95fb44527ed 100644 --- a/system/netstack/shard/tests/udp.rs +++ b/system/netstack/shard/tests/udp.rs @@ -9,7 +9,7 @@ use std::net::Ipv4Addr; use std::time::Duration; use common::*; -use toyos_net_shard::{Event, Refusal, Shard}; +use toyos_net_shard::{Event, IfIndex, Refusal, Shard}; use toyos_net_testnet::Fate; use toyos_net_ip::Nud; use toyos_net_udp::{Counter, Error, SocketError}; @@ -42,7 +42,7 @@ fn s_udp_us_046_shard_a_port_unreachable_refuses_the_connected_socket_once() { net.advance(Duration::from_millis(1)); let datagram = net.wire()[start..].iter().find(|c| c.from == a && name(&c.frame).starts_with("IPv4 Udp")).expect("the datagram left").frame.clone(); let now = net.now(); - net.nodes[a].shard.receive(now, &icmp_about(&datagram, 3, 3, [0; 4])); + net.nodes[a].receive(now, &icmp_about(&datagram, 3, 3, [0; 4])); let mut buf = [0u8; 16]; assert_eq!(net.nodes[a].shard.recv_from(socket, &mut buf), Err(Error::Failed(SocketError::Refused))); assert_eq!(net.nodes[a].shard.recv_from(socket, &mut buf), Ok(None)); @@ -85,8 +85,8 @@ fn b_resolved() -> (toyos_net_testnet::Net, usize, toyos_net_udp::SocketId) { let socket = net.nodes[a].shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5000)), || 0).unwrap(); net.nodes[a].shard.send_to(now, socket, B, 9, b"hello").unwrap(); net.advance(Duration::from_millis(1)); - let shard = &net.nodes[a].shard; - assert!(shard.ip().neighbour(shard.iface(), B).is_some_and(|entry| entry.mac().is_some()) && shard.ip().neighbour(shard.iface(), C).is_none()); + let (shard, iface) = (&net.nodes[a].shard, net.nodes[a].iface); + assert!(shard.ip().neighbour(iface, B).is_some_and(|entry| entry.mac().is_some()) && shard.ip().neighbour(iface, C).is_none()); (net, a, socket) } @@ -94,7 +94,7 @@ fn b_resolved() -> (toyos_net_testnet::Net, usize, toyos_net_udp::SocketId) { /// destination and payload, any other frame by its name. fn left(shard: &mut Shard, now: Instant) -> Vec { let mut left = Vec::new(); - shard.transmit(now, usize::MAX, |bytes| { + shard.transmit(now, usize::MAX, |_, bytes| { let frame = Frame::parse(bytes).expect("a frame"); let udp = Ipv4Packet::parse(frame.body()).ok().and_then(|ip| UdpDatagram::parse(&ip).ok().map(|udp| format!("UDP {} {}", ip.destination(), String::from_utf8_lossy(udp.payload())))); left.push(udp.unwrap_or_else(|| name(bytes))); @@ -103,8 +103,8 @@ fn left(shard: &mut Shard, now: Instant) -> Vec { } /// How many datagrams [ip] holds for C while it asks for its link address. -fn held_by_ip(shard: &Shard) -> Option { - match shard.ip().neighbour(shard.iface(), C) { +fn held_by_ip(shard: &Shard, iface: IfIndex) -> Option { + match shard.ip().neighbour(iface, C) { Some(Nud::Incomplete(asking)) => Some(asking.pending.queued()), _ => None, } @@ -117,14 +117,14 @@ fn held_by_ip(shard: &Shard) -> Option { fn s_udp_us_026_shard_a_datagram_waits_in_its_socket_for_its_next_hop_and_holds_back_no_other() { let (mut net, a, socket) = b_resolved(); let now = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; for (destination, data) in [(C, "c1"), (B, "b1"), (C, "c2")] { shard.send_to(now, socket, destination, 9, data.as_bytes()).unwrap(); } assert_eq!(left(shard, now), ["UDP 192.0.2.2 b1", "ARP request 192.0.2.3"]); - assert_eq!(held_by_ip(shard), Some(0), "[ip] holds no sender's datagram"); + assert_eq!(held_by_ip(shard, iface), Some(0), "[ip] holds no sender's datagram"); assert_eq!(left(shard, now), [""; 0], "what waits asks for nothing again"); - shard.receive(now, &c_answers()); + shard.receive(now, iface, &c_answers()); assert_eq!(left(shard, now), ["UDP 192.0.2.3 c1", "UDP 192.0.2.3 c2"]); } @@ -135,7 +135,7 @@ fn s_udp_us_026_shard_a_datagram_waits_in_its_socket_for_its_next_hop_and_holds_ fn s_udp_us_025_shard_what_waits_for_a_next_hop_is_bounded_by_its_sockets_queue() { let (mut net, a, socket) = b_resolved(); let now = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; let other = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5002)), || 0).unwrap(); let per = toyos_net_udp::limits::TX_DATAGRAMS; for n in 0..per { @@ -146,8 +146,8 @@ fn s_udp_us_025_shard_what_waits_for_a_next_hop_is_bounded_by_its_sockets_queue( for destination in [C, B] { assert_eq!(shard.send_to(now, socket, destination, 9, b"x"), Err(Error::Refused(Counter::TxQueueFull)), "{destination}"); } - assert_eq!(held_by_ip(shard), Some(0)); - shard.receive(now, &c_answers()); + assert_eq!(held_by_ip(shard, iface), Some(0)); + shard.receive(now, iface, &c_answers()); let out = left(shard, now); for (sender, tag) in [(socket, 's'), (other, 'o')] { let of: Vec<&String> = out.iter().filter(|frame| frame.starts_with(&format!("UDP 192.0.2.3 {tag}"))).collect(); @@ -176,7 +176,7 @@ fn s_udp_us_027_shard_a_failed_next_hop_drops_what_waited_for_it_and_tells_no_un assert_eq!((shard.udp_counters().get(Counter::TxUnreachable), shard.ip().counters().get(toyos_net_ip::Counter::NbFailed)), (2, 1)); assert_eq!(shard.recv_from(socket, &mut [0; 16]), Ok(None)); let now = net.now(); - net.nodes[a].shard.receive(now, &c_answers()); + net.nodes[a].receive(now, &c_answers()); net.advance(Duration::from_millis(1)); assert!(net.wire()[start..].iter().all(|carried| !name(&carried.frame).starts_with("IPv4 Udp")), "{}", dump(&net, start)); } @@ -188,13 +188,13 @@ fn s_udp_us_027_shard_a_failed_next_hop_drops_what_waited_for_it_and_tells_no_un fn s_udp_us_024_shard_a_waiting_datagram_whose_route_goes_is_dropped() { let (mut net, a, socket) = b_resolved(); let now = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; shard.send_to(now, socket, C, 9, b"c1").unwrap(); assert_eq!(left(shard, now), ["ARP request 192.0.2.3"]); - shard.link_down(now).unwrap(); + shard.link_down(now, iface).unwrap(); assert_eq!((left(shard, now), shard.udp_counters().get(Counter::TxUnreachable), shard.ip().counters().get(toyos_net_ip::Counter::RouteNoSourceAddress)), (vec![], 0, 1)); - shard.link_up(now).unwrap(); - shard.receive(now, &c_answers()); + shard.link_up(now, iface).unwrap(); + shard.receive(now, iface, &c_answers()); assert!(left(shard, now).iter().all(|frame| !frame.starts_with("UDP"))); } @@ -205,14 +205,14 @@ fn s_udp_us_024_shard_a_waiting_datagram_whose_route_goes_is_dropped() { fn a_waiting_datagram_whose_link_returns_before_an_opportunity_waits_anew_and_leaves() { let (mut net, a, socket) = b_resolved(); let now = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; shard.send_to(now, socket, C, 9, b"c1").unwrap(); assert_eq!(left(shard, now), ["ARP request 192.0.2.3"]); - shard.link_down(now).unwrap(); - shard.link_up(now).unwrap(); + shard.link_down(now, iface).unwrap(); + shard.link_up(now, iface).unwrap(); assert_eq!(left(shard, now), ["ARP request 192.0.2.1", "ARP request 192.0.2.3"], "the address announced, and the hop asked for anew"); assert_eq!((shard.udp_counters().get(Counter::TxUnreachable), shard.ip().counters().get(toyos_net_ip::Counter::RouteNoSourceAddress)), (0, 0)); - shard.receive(now, &c_answers()); + shard.receive(now, iface, &c_answers()); assert_eq!(left(shard, now), ["UDP 192.0.2.3 c1"]); } @@ -227,7 +227,7 @@ const ROUTER_IN_25: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 126); /// and, of a UDP datagram, its destination and source port; any other frame by its name. fn opportunity(shard: &mut Shard, now: Instant) -> Vec<(MacAddr, String)> { let mut left = Vec::new(); - shard.transmit(now, usize::MAX, |bytes| { + shard.transmit(now, usize::MAX, |_, bytes| { let frame = Frame::parse(bytes).expect("a frame"); let udp = Ipv4Packet::parse(frame.body()).ok().and_then(|ip| UdpDatagram::parse(&ip).ok().map(|udp| format!("UDP {} from port {}", ip.destination(), udp.source_port().map_or(0, |p| p.get())))); left.push((frame.destination(), udp.unwrap_or_else(|| name(bytes)))); @@ -237,11 +237,11 @@ fn opportunity(shard: &mut Shard, now: Instant) -> Vec<(MacAddr, String)> { /// [ip] counted `count` datagrams refused a link broadcast, and the shard's one line for the /// rule names `destination`. -fn refused_a_broadcast(shard: &mut Shard, destination: Ipv4Addr, count: u64) { +fn refused_a_broadcast(shard: &mut Shard, iface: IfIndex, destination: Ipv4Addr, count: u64) { let rule = toyos_net_ip::Counter::IpBroadcastNotPermitted; assert_eq!(shard.ip().counters().get(rule), count); let lines: Vec = shard.drain_events().filter(|event| matches!(event, Event::Refused { .. })).collect(); - let refusal = toyos_net_ip::Refusal { rule, iface: shard.iface(), peer: toyos_net_ip::Peer::Ip(destination) }; + let refusal = toyos_net_ip::Refusal { rule, iface, peer: toyos_net_ip::Peer::Ip(destination) }; assert_eq!(lines, [Event::Refused { refusal: Refusal::Ip(refusal), suppressed: 0 }]); } @@ -256,16 +256,16 @@ fn s_udp_us_021_shard_a_datagram_accepted_for_a_host_is_no_broadcast_after_a_ren for closed in [false, true] { let (mut net, a, _) = segment(); let now = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; shard.drain_events().for_each(drop); let socket = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5000)), || 0).unwrap(); assert_eq!(shard.send_to(now, socket, EDGE_OF_25, 5001, b"hi"), Ok(()), "one host of the /24"); if closed { shard.udp_close(now, socket).unwrap(); } - assert_eq!(shard.add_address(now, A, 25), Ok(()), "the renewal"); + assert_eq!(shard.add_address(now, iface, A, 25), Ok(()), "the renewal"); assert_eq!(opportunity(shard, now), [], "closed: {closed}"); - refused_a_broadcast(shard, EDGE_OF_25, 1); + refused_a_broadcast(shard, iface, EDGE_OF_25, 1); assert_eq!(opportunity(shard, now), [], "and it is gone, not waiting"); } } @@ -276,15 +276,15 @@ fn s_udp_us_021_shard_a_datagram_accepted_for_a_host_is_no_broadcast_after_a_ren fn s_udp_us_021_shard_a_datagram_accepted_for_the_router_is_no_broadcast_after_a_renewal_widens_the_prefix() { let (mut net, a, _) = segment(); let now = net.now(); - let shard = &mut net.nodes[a].shard; - shard.add_address(now, A, 25).unwrap(); - shard.set_gateways(now, &[ROUTER_IN_25]).unwrap(); + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; + shard.add_address(now, iface, A, 25).unwrap(); + shard.set_gateways(now, iface, &[ROUTER_IN_25]).unwrap(); shard.drain_events().for_each(drop); let socket = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5000)), || 0).unwrap(); assert_eq!(shard.send_to(now, socket, EDGE_OF_24, 5001, b"hi"), Ok(()), "a host behind the router"); - assert_eq!(shard.add_address(now, A, 24), Ok(()), "the renewal"); + assert_eq!(shard.add_address(now, iface, A, 24), Ok(()), "the renewal"); assert_eq!(opportunity(shard, now), []); - refused_a_broadcast(shard, EDGE_OF_24, 1); + refused_a_broadcast(shard, iface, EDGE_OF_24, 1); } // US-21 for a datagram that waited: accepted for one host of the /24 whose link address nobody @@ -295,7 +295,7 @@ fn s_udp_us_021_shard_a_datagram_accepted_for_the_router_is_no_broadcast_after_a fn s_udp_us_021_shard_a_datagram_that_waited_for_a_host_is_asked_its_route_again_and_keeps_its_permission() { let (mut net, a, _) = segment(); let now = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; shard.drain_events().for_each(drop); let permitted = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5000)), || 0).unwrap(); let unpermitted = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5002)), || 0).unwrap(); @@ -305,9 +305,9 @@ fn s_udp_us_021_shard_a_datagram_that_waited_for_a_host_is_asked_its_route_again } shard.udp_set_broadcast(permitted, false).unwrap(); assert_eq!(opportunity(shard, now), [(MacAddr::BROADCAST, "ARP request 192.0.2.127".to_string())], "both wait for the host"); - assert_eq!(shard.add_address(now, A, 25), Ok(()), "the renewal"); + assert_eq!(shard.add_address(now, iface, A, 25), Ok(()), "the renewal"); assert_eq!(opportunity(shard, now), [(MacAddr::BROADCAST, "UDP 192.0.2.127 from port 5000".to_string())]); - refused_a_broadcast(shard, EDGE_OF_25, 1); + refused_a_broadcast(shard, iface, EDGE_OF_25, 1); } // The permission is the one the socket held at the call (POSIX's `SO_BROADCAST` is read by the @@ -317,7 +317,7 @@ fn s_udp_us_021_shard_a_datagram_that_waited_for_a_host_is_asked_its_route_again fn s_udp_us_021_shard_a_datagram_carries_the_permission_of_its_call() { let (mut net, a, _) = segment(); let now = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; shard.drain_events().for_each(drop); let permitted = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5000)), || 0).unwrap(); let unpermitted = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5002)), || 0).unwrap(); @@ -327,7 +327,7 @@ fn s_udp_us_021_shard_a_datagram_carries_the_permission_of_its_call() { } shard.udp_set_broadcast(permitted, false).unwrap(); shard.udp_set_broadcast(unpermitted, true).unwrap(); - assert_eq!(shard.add_address(now, A, 25), Ok(()), "the renewal"); + assert_eq!(shard.add_address(now, iface, A, 25), Ok(()), "the renewal"); assert_eq!(opportunity(shard, now), [(MacAddr::BROADCAST, "UDP 192.0.2.127 from port 5000".to_string())]); - refused_a_broadcast(shard, EDGE_OF_25, 1); + refused_a_broadcast(shard, iface, EDGE_OF_25, 1); } diff --git a/system/netstack/udp/src/lib.rs b/system/netstack/udp/src/lib.rs index b948ef8619d..5b47f0abac9 100644 --- a/system/netstack/udp/src/lib.rs +++ b/system/netstack/udp/src/lib.rs @@ -42,7 +42,7 @@ use alloc::collections::{BTreeMap, VecDeque}; use alloc::vec::Vec; use core::net::Ipv4Addr; -use toyos_net_ip::{Arrival, Cast, ErrorClass, Flow, Ip, Source, Transport, TransportError, UdpOut}; +use toyos_net_ip::{Arrival, Cast, ErrorClass, Flow, IfIndex, Ip, Source, Transport, TransportError, UdpOut}; use toyos_net_wire::addr::is_martian; use toyos_net_wire::ethernet::MacClass; use toyos_net_wire::ipv4::Ttl; @@ -158,6 +158,8 @@ pub struct Received { pub destination: Ipv4Addr, pub ttl: u8, pub link_broadcast: bool, + /// The interface it arrived by. + pub iface: IfIndex, pub at: Instant, } @@ -189,9 +191,18 @@ struct Stored { destination: Ipv4Addr, ttl: u8, link_broadcast: bool, + iface: IfIndex, at: Instant, } +/// Where the acquisition socket's datagram is from: the source it names, and the interface whose +/// address it acquires, which alone it leaves by. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Origin { + pub iface: IfIndex, + pub source: Ipv4Addr, +} + /// A datagram accepted for sending, with the addressing it was accepted with. #[derive(Debug)] struct Queued { @@ -205,7 +216,10 @@ struct Queued { /// leaves is decided by this, not by what the socket holds then. broadcast: bool, payload: Vec, - /// The next hop whose link address it waits for: not offered until [`Udp::wake`] names it. + /// The interface the send named, which alone it may leave by. + iface: Option, + /// The next hop whose link address it waits for, or the hop of a link with no room: not + /// offered until [`Udp::wake`] names it. waits: Option, } @@ -216,6 +230,7 @@ impl Queued { destination: self.destination, ttl: self.ttl, broadcast: self.broadcast, + iface: self.iface, datagram: UdpBuilder { source: self.source_port, destination: self.port, data: &self.payload }, } } @@ -520,11 +535,13 @@ impl Udp { /// The acquisition socket's send, naming its source: 0.0.0.0 to the limited broadcast, or an /// assigned address. - pub fn send_from(&mut self, ip: &mut Ip, id: SocketId, source: Ipv4Addr, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), Error> { - self.submit(ip, id, Some(source), destination, port, payload) + pub fn send_from(&mut self, ip: &mut Ip, id: SocketId, origin: Origin, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), Error> { + self.submit(ip, id, Some((origin.source, origin.iface)), destination, port, payload) } - fn submit(&mut self, ip: &mut Ip, id: SocketId, named: Option, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), Error> { + fn submit(&mut self, ip: &mut Ip, id: SocketId, named: Option<(Ipv4Addr, IfIndex)>, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), Error> { + let iface = named.map(|(_, iface)| iface); + let named = named.map(|(source, _)| source); let socket = self.socket(id)?; if let Some(error) = socket.pending.take() { return Err(Error::Failed(error)); @@ -572,7 +589,7 @@ impl Udp { } (None, Binding::Any) => (Ipv4Addr::UNSPECIFIED, Source::Any), }; - let source = match ip.route(destination, lookup, None) { + let source = match ip.route(destination, lookup, iface) { Ok(route) if lookup == Source::Any => route.source, Ok(_) => source, Err(refusal) => return self.refuse(route_refusal(refusal), me, peer), @@ -582,7 +599,7 @@ impl Udp { return self.refuse(Counter::TxQueueFull, me, peer); } let ttl = if destination.is_multicast() { multicast_ttl } else { ttl }; - socket.tx.push_back(Queued { source, source_port: local_port, destination, port, ttl, broadcast: broadcast_permitted, payload: payload.to_vec(), waits: None }); + socket.tx.push_back(Queued { source, source_port: local_port, destination, port, ttl, broadcast: broadcast_permitted, payload: payload.to_vec(), iface, waits: None }); if !core::mem::replace(&mut socket.offered, true) { self.eligible.push(Sender::Socket(id)); } @@ -611,6 +628,7 @@ impl Udp { destination: d.destination, ttl: d.ttl, link_broadcast: d.link_broadcast, + iface: d.iface, at: d.at, })) } @@ -689,6 +707,7 @@ impl Udp { destination, ttl: arrival.packet.ttl(), link_broadcast: arrival.link == MacClass::Broadcast, + iface: arrival.iface, at: now, }); self.count(Counter::RxDelivered); diff --git a/system/netstack/udp/tests/common/mod.rs b/system/netstack/udp/tests/common/mod.rs index 180e24b78cb..1193b7c5e78 100644 --- a/system/netstack/udp/tests/common/mod.rs +++ b/system/netstack/udp/tests/common/mod.rs @@ -189,7 +189,7 @@ impl U { /// A's interface up with MAC A and nothing configured. pub fn bare() -> Self { let mut ip = Ip::new(Instant::from_nanos(0), [0x5a; 16]); - let if0 = ip.add_interface(Instant::from_nanos(0), IndividualMac::new(MAC_A).unwrap()); + let if0 = ip.add_interface(Instant::from_nanos(0), IndividualMac::new(MAC_A).unwrap(), toyos_net_ip::Rank(0)); ip.link_up(Instant::from_nanos(0), if0).unwrap(); ip.join(Instant::from_nanos(0), if0, MulticastAddr::new(MDNS).unwrap()).unwrap(); Self::setup_until(&mut ip, Self::instant(0)); @@ -203,7 +203,7 @@ impl U { pub fn uf() -> Self { let mut ip = Ip::new(Instant::from_nanos(0), [0x5a; 16]); let t = Instant::from_nanos(0); - let if0 = ip.add_interface(t, IndividualMac::new(MAC_A).unwrap()); + let if0 = ip.add_interface(t, IndividualMac::new(MAC_A).unwrap(), toyos_net_ip::Rank(0)); ip.link_up(t, if0).unwrap(); ip.add_address(t, if0, A, 24).unwrap(); Self::setup_until(&mut ip, Instant::from_millis(3_000)); @@ -308,13 +308,14 @@ impl U { let mut framed = false; let served = udp.serve(sender, |out| { let mut buf = [0u8; FRAME]; - match ip.send_udp(now, out, &mut buf) { - Ok(Sent::Frame(n)) => { + match ip.send_udp(now, &usize::MAX, out, &mut buf) { + Ok(Sent::Frame { len: n, .. }) => { frames.push(buf[..n].to_vec()); framed = true; Offer::Taken } Ok(Sent::Pending(next_hop)) => Offer::Waits(next_hop), + Ok(Sent::NoRoom { .. }) => unreachable!("every interface has room"), Err(_) => Offer::Taken, } }); diff --git a/system/netstack/udp/tests/send.rs b/system/netstack/udp/tests/send.rs index 72b7c702271..b5554244af6 100644 --- a/system/netstack/udp/tests/send.rs +++ b/system/netstack/udp/tests/send.rs @@ -458,19 +458,19 @@ fn acquisition() -> (U, toyos_net_udp::SocketId) { #[test] fn s_udp_us_028_only_the_acquisition_socket_sends_from_nowhere() { let (mut u, id) = acquisition(); - u.udp.send_from(&mut u.ip, id, ANY, LIMITED, 67, b"discover").unwrap(); + u.udp.send_from(&mut u.ip, id, toyos_net_udp::Origin { iface: u.if0, source: ANY }, LIMITED, 67, b"discover").unwrap(); let out = u.out(); assert_eq!(ip_of(&out[0]).unwrap().source(), ANY); - assert_eq!(u.udp.send_from(&mut u.ip, id, ANY, R, 67, b"x"), refused(Counter::SourceNotPermitted)); + assert_eq!(u.udp.send_from(&mut u.ip, id, toyos_net_udp::Origin { iface: u.if0, source: ANY }, R, 67, b"x"), refused(Counter::SourceNotPermitted)); let other = u.bind(ANY, 5_000).unwrap(); u.udp.set_broadcast(other, true).unwrap(); - assert_eq!(u.udp.send_from(&mut u.ip, other, ANY, LIMITED, 67, b"x"), refused(Counter::SourceNotPermitted)); + assert_eq!(u.udp.send_from(&mut u.ip, other, toyos_net_udp::Origin { iface: u.if0, source: ANY }, LIMITED, 67, b"x"), refused(Counter::SourceNotPermitted)); } #[test] fn s_udp_us_029_the_discover_byte_for_byte() { let (mut u, id) = acquisition(); let discover = hex(V_DHCP_DISCOVER); - u.udp.send_from(&mut u.ip, id, ANY, LIMITED, 67, &discover[42..]).unwrap(); + u.udp.send_from(&mut u.ip, id, toyos_net_udp::Origin { iface: u.if0, source: ANY }, LIMITED, 67, &discover[42..]).unwrap(); assert_eq!(u.out(), [discover]); } diff --git a/system/virtio-net/src/nic.rs b/system/virtio-net/src/nic.rs index ea49a439964..469be896e22 100644 --- a/system/virtio-net/src/nic.rs +++ b/system/virtio-net/src/nic.rs @@ -177,6 +177,8 @@ impl VirtioNet { } impl toyos_link::driver::Nic for VirtioNet { + const MEDIUM: toyos_link::wire::Medium = toyos_link::wire::Medium::Wired; + fn mac(&self) -> [u8; 6] { self.mac } From 1e8ea11ed8cc7ac6eb5661436f5c54c0d6f953de Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 31 Oct 2026 19:24:14 +0100 Subject: [PATCH 2/9] netstack's node holds a part per link: its lease, its DHCP client, its name's responder The P5 build's second half below netstack itself, which follows. - node: built with no link and serving from there; add_link gives each link its own DHCP client, lease state and mDNS responder, under one host name. Frames come in and go out by link, and one transmit opportunity takes each link's credit. - lease: one state per link, written by this module alone as before. The links' one acquisition socket hands each datagram to the client of the link it came by, and each client's message leaves by its own link. Lookups ask the asked link's resolvers: of the links holding a lease, one up before one down, then by rank and arrival. One cache holds their answers, emptied with every state written, every link down and every change of the link asked. - name: one socket on 5353, one responder per link; a message is its arrival link's responder's, and what a responder sends leaves by its link alone (udp's send_via). A name lost on one link is that link's. - A stream from an address the machine lost ends ADDRESS_GONE; the SDK's Refused::AddressGone, std's AddrNotAvailable and libc's EADDRNOTAVAIL name it. Co-Authored-By: Claude Opus 5.5 --- sdk/libc/src/errcode.rs | 2 + sdk/std/sys/net/connection.rs | 1 + sdk/toyos/src/net/stream.rs | 11 +- system/netstack/node/src/lease.rs | 291 +++++++++++-------- system/netstack/node/src/lib.rs | 188 +++++++----- system/netstack/node/src/name.rs | 98 ++++--- system/netstack/node/src/resolve.rs | 4 +- system/netstack/node/src/streams.rs | 3 +- system/netstack/node/tests/common/mod.rs | 27 +- system/netstack/node/tests/datagram.rs | 6 +- system/netstack/node/tests/lan/mod.rs | 17 +- system/netstack/node/tests/lease.rs | 68 ++--- system/netstack/node/tests/links.rs | 355 +++++++++++++++++++++++ system/netstack/node/tests/listeners.rs | 11 +- system/netstack/node/tests/name.rs | 12 +- system/netstack/node/tests/resolve.rs | 26 +- system/netstack/node/tests/slirp.rs | 10 +- system/netstack/node/tests/streams.rs | 38 ++- system/netstack/shard/src/lib.rs | 7 + system/netstack/src/serve.rs | 2 + system/netstack/udp/src/lib.rs | 18 +- 21 files changed, 849 insertions(+), 346 deletions(-) create mode 100644 system/netstack/node/tests/links.rs diff --git a/sdk/libc/src/errcode.rs b/sdk/libc/src/errcode.rs index 2eaa6163f7e..d4ad92a6e2d 100644 --- a/sdk/libc/src/errcode.rs +++ b/sdk/libc/src/errcode.rs @@ -52,6 +52,7 @@ pub(crate) const ENOTCONN: i32 = 107; pub(crate) const ETIMEDOUT: i32 = 110; pub(crate) const ECONNREFUSED: i32 = 111; pub(crate) const EHOSTUNREACH: i32 = 113; +pub(crate) const EADDRNOTAVAIL: i32 = 99; pub(crate) const ESTALE: i32 = 116; /// The errno a kernel refusal is answered with. @@ -111,6 +112,7 @@ pub(crate) fn of_stream(refused: toyos::net::stream::Refused) -> i32 { Refused::TimedOut | Refused::ConnectionTimedOut => ETIMEDOUT, Refused::Reset => ECONNRESET, Refused::Unreachable => EHOSTUNREACH, + Refused::AddressGone => EADDRNOTAVAIL, Refused::Aborted => ECONNABORTED, Refused::Shut => EPIPE, } diff --git a/sdk/std/sys/net/connection.rs b/sdk/std/sys/net/connection.rs index 680e1475680..3bef6d9d3de 100644 --- a/sdk/std/sys/net/connection.rs +++ b/sdk/std/sys/net/connection.rs @@ -93,6 +93,7 @@ pub fn stream_refused(refusal: Refused) -> io::Error { Refused::Reset => io::const_error!(io::ErrorKind::ConnectionReset, "connection reset"), Refused::ConnectionTimedOut => io::const_error!(io::ErrorKind::TimedOut, "the connection timed out"), Refused::Unreachable => io::const_error!(io::ErrorKind::HostUnreachable, "the peer is unreachable"), + Refused::AddressGone => io::const_error!(io::ErrorKind::AddrNotAvailable, "this machine gave up the connection's address"), Refused::Aborted => io::const_error!(io::ErrorKind::ConnectionAborted, "netstack ended the stream"), Refused::Shut => SHUT_DOWN, } diff --git a/sdk/toyos/src/net/stream.rs b/sdk/toyos/src/net/stream.rs index 94db3a7b9b8..454b391e415 100644 --- a/sdk/toyos/src/net/stream.rs +++ b/sdk/toyos/src/net/stream.rs @@ -4,7 +4,8 @@ //! with the connect or the accept. [`LAYOUT`] is its one declaration: a header //! of [`WORDS`] words, each on a 64-byte line of its own, then the receive ring //! netstack fills and the send ring the client fills. The receive ring's `end` -//! is the peer's FIN, [`RESET`], [`TIMED_OUT`] or [`UNREACHABLE`]; the send +//! is the peer's FIN, [`RESET`], [`TIMED_OUT`], [`UNREACHABLE`] or +//! [`ADDRESS_GONE`]; the send //! ring's is the client's shutdown; [`NODELAY`] is the client's Nagle switch //! (RFC 9293 §3.7.4). The rest of the header is the session's: the gates, the //! receiving half's shutdown and the hang-up any holder saw. @@ -111,6 +112,9 @@ pub const RESET: u32 = 2; pub const TIMED_OUT: u32 = 3; /// The receive ring's `end`: an ICMP error ended the connection. pub const UNREACHABLE: u32 = 4; +/// The receive ring's `end`: this machine gave up the connection's local +/// address, its link's lease ended or another host took it. +pub const ADDRESS_GONE: u32 = 5; // ---- a holder's connection ---- @@ -239,6 +243,8 @@ pub enum Refused { ConnectionTimedOut, /// An ICMP error ended the connection. Unreachable, + /// This machine gave up the connection's local address. + AddressGone, /// netstack went away, or wrote what no netstack writes. Aborted, /// The sending half was shut down. @@ -259,6 +265,7 @@ impl Refused { Self::Unreachable => 4, Self::Aborted => 5, Self::Shut => 6, + Self::AddressGone => 7, }) } @@ -271,6 +278,7 @@ impl Refused { 4 => Self::Unreachable, 5 => Self::Aborted, 6 => Self::Shut, + 7 => Self::AddressGone, _ => return None, }) } @@ -283,6 +291,7 @@ impl From for Refused { Answer::TimedOut => Self::TimedOut, Answer::Failed(TIMED_OUT) => Self::ConnectionTimedOut, Answer::Failed(UNREACHABLE) => Self::Unreachable, + Answer::Failed(ADDRESS_GONE) => Self::AddressGone, Answer::Failed(_) => Self::Reset, Answer::Shut => Self::Shut, Answer::Gone | Answer::Violation(_) => Self::Aborted, diff --git a/system/netstack/node/src/lease.rs b/system/netstack/node/src/lease.rs index f063d6a9315..cfe12b47ffe 100644 --- a/system/netstack/node/src/lease.rs +++ b/system/netstack/node/src/lease.rs @@ -1,62 +1,33 @@ -//! The interface's address, gateway and resolvers as one state, and its one writer. +//! Each link's address, gateway and resolvers as one state, and its one writer. //! -//! The state is `Unaddressed`, `Probing` an address a server acknowledged, or `Held`: a lease -//! whose address [ip] verified. The shard is private to this module, so nothing else in the crate -//! reaches `add_address`, `remove_address` or `set_gateways`: +//! A link's state is `Unaddressed`, `Probing` an address a server acknowledged, or `Held`: a +//! lease whose address [ip] verified. The shard is private to this module, so nothing else in the +//! crate reaches `add_address`, `remove_address` or `set_gateways`: //! -//! - the gateway is written only with a held lease, and [ip] withdraws it with the address, so no -//! route outlives its address; +//! - a link's gateway is written only with its held lease, and [ip] withdraws it with the +//! address, so no route outlives its address; //! - a lease is held only against a [`Verified`], which only [`Stack::report`] makes, from [ip]'s -//! report for the address under probe, so no address is in use before conflict detection ends; -//! - the resolvers are the held lease's, so they come and go with it, and the answers they gave -//! go with every write of the state and every link that goes down ([`network::Network`]): after -//! either the next lookup may be asking another network, so none is answered from this one's. +//! report for the link's address under probe, so no address is in use before conflict +//! detection ends; +//! - the resolvers lookups ask are one link's held lease's: of the links holding one, the first +//! up, by rank and then arrival ([`Stack::asked`]). The answers they gave go with every write of any +//! link's state, every link that goes down and every change of the link asked: after any of +//! them the next lookup may be asking another network, so none is answered from this one's. //! -//! [ip] reports on no address but the one this module added, and one report at a time: each is -//! read against the state as it stands. +//! [ip] reports on no address but those this module added, and one report at a time: each is +//! read against its link's state as it stands. mod tcp; -mod network { - use toyos_dns::Cache; - - use super::State; - - /// The state and the answers its resolvers gave, written together: the state only by - /// [`Network::set`], which empties the answers as it writes it. - pub(super) struct Network { - state: State, - answers: Cache, - } - - impl Network { - pub(super) fn new() -> Self { - Self { state: State::Unaddressed, answers: Cache::default() } - } - - pub(super) fn state(&self) -> &State { - &self.state - } - - pub(super) fn set(&mut self, state: State) { - self.state = state; - self.answers.empty(); - } - - pub(super) fn answers(&mut self) -> &mut Cache { - &mut self.answers - } - } -} - use alloc::collections::VecDeque; +use alloc::vec::Vec; use core::net::Ipv4Addr; use toyos_dhcp::{Lease, Transmission}; use toyos_dns::Cache; -use toyos_net_shard::{Config, Event, Refusal, Shard}; +use toyos_net_shard::{Config, Credit, Event, IfIndex, Origin, Rank, Refusal, Shard}; use toyos_net_udp::{Received, SocketId}; -use toyos_net_wire::ethernet::MacAddr; +use toyos_net_wire::ethernet::{IndividualMac, MacAddr}; use toyos_net_wire::ipv4::{MulticastAddr, Ttl}; use toyos_net_wire::{Instant, Port}; @@ -71,37 +42,49 @@ enum State { Held(Lease), } -/// Proof that [ip] finished conflict detection for the address under probe. +/// Proof that [ip] finished conflict detection for a link's address under probe. pub(crate) struct Verified(Ipv4Addr); -/// What [ip] said became of an address, one event at a time. +/// What [ip] said became of a link's address, one event at a time. pub(crate) enum Report { Verified(Verified), - /// Another host holds the address, at this MAC: [ip] removed it, and the state is + /// Another host holds the address, at this MAC: [ip] removed it, and the link's state is /// `Unaddressed`. Conflict(MacAddr), - /// The link went down under the probe: [ip] removed the address, and the state is + /// The link went down under the probe: [ip] removed the address, and the link's state is /// `Unaddressed`. NotVerified, } +/// One link's part. +struct Link { + iface: IfIndex, + rank: Rank, + up: bool, + state: State, +} + pub(crate) struct Stack { shard: Shard, - /// The DHCP client's socket. + /// The DHCP clients' one socket, each datagram for the client of the link it came by. socket: SocketId, - network: network::Network, + /// In the order they were added. + links: Vec, + /// What the asked link's resolvers answered: emptied with every write of a link's state, + /// every link that goes down and every change of the link asked. + answers: Cache, /// What [ip] reported of an address and [`Stack::report`] has not read: never a log line. - inbox: VecDeque, + inbox: VecDeque<(IfIndex, Fate)>, } -/// [ip]'s report on the address this module added. +/// [ip]'s report on an address this module added. enum Fate { Verified(Ipv4Addr), /// Removed by [ip]: for the host at this MAC, or with none, for a link that went down. Taken(Ipv4Addr, Option), } -/// [ip] refuses nothing about the one interface and the one address this module gave it. +/// [ip] refuses nothing about an interface and an address this module gave it. fn agreed(answer: Result) -> T { match answer { Ok(value) => value, @@ -110,39 +93,83 @@ fn agreed(answer: Result) -> T { } impl Stack { - /// One interface, link down, unaddressed. + /// No link yet. pub(crate) fn new(now: Instant, config: Config) -> Result { let mut shard = Shard::new(now, config)?; let Ok(socket) = shard.acquisition() else { unreachable!("a new shard has no socket on port 68") }; - Ok(Self { shard, socket, network: network::Network::new(), inbox: VecDeque::new() }) + Ok(Self { shard, socket, links: Vec::new(), answers: Cache::default(), inbox: VecDeque::new() }) + } + + /// A link's interface, down and unaddressed. + pub(crate) fn add_link(&mut self, now: Instant, mac: IndividualMac, rank: Rank) -> IfIndex { + let iface = self.shard.add_interface(now, mac, rank); + self.links.push(Link { iface, rank, up: false, state: State::Unaddressed }); + iface } pub(crate) fn shard(&self) -> &Shard { &self.shard } - pub(crate) fn lease(&self) -> Option<&Lease> { - match self.network.state() { + /// Every link, in the order added. + pub(crate) fn links(&self) -> impl Iterator + '_ { + self.links.iter().map(|link| link.iface) + } + + fn link_of(&self, iface: IfIndex) -> &Link { + self.links.iter().find(|link| link.iface == iface).unwrap_or_else(|| unreachable!("{iface:?} is no link of the node's")) + } + + fn link_mut(&mut self, iface: IfIndex) -> &mut Link { + self.links.iter_mut().find(|link| link.iface == iface).unwrap_or_else(|| unreachable!("{iface:?} is no link of the node's")) + } + + pub(crate) fn up(&self, iface: IfIndex) -> bool { + self.link_of(iface).up + } + + /// `iface`'s held lease. + pub(crate) fn lease(&self, iface: IfIndex) -> Option<&Lease> { + match &self.link_of(iface).state { State::Held(lease) => Some(lease), State::Unaddressed | State::Probing(_) => None, } } - fn address(&self) -> Option { - match self.network.state() { + /// The link lookups ask, of those holding a lease: one that is up before one that is not, + /// and then the first by rank and by when it was added. A link down keeps its lease, so with + /// none up the lookups keep asking the one they asked. + pub(crate) fn asked(&self) -> Option { + let held = |link: &&Link| matches!(link.state, State::Held(_)); + self.links.iter().filter(held).min_by_key(|link| (!link.up, link.rank)).map(|link| link.iface) + } + + /// The asked link's lease, whose resolvers lookups ask. + pub(crate) fn asked_lease(&self) -> Option<&Lease> { + self.asked().and_then(|iface| self.lease(iface)) + } + + fn address(&self, iface: IfIndex) -> Option { + match &self.link_of(iface).state { State::Unaddressed => None, State::Probing(address) => Some(*address), State::Held(lease) => Some(lease.address), } } + /// Writes `iface`'s state; the answers go with it. + fn set(&mut self, iface: IfIndex, state: State) { + self.link_mut(iface).state = state; + self.answers.empty(); + } + // ---- the device and the clock ---- - pub(crate) fn receive(&mut self, now: Instant, frame: &[u8]) { - self.shard.receive(now, frame); + pub(crate) fn receive(&mut self, now: Instant, iface: IfIndex, frame: &[u8]) { + self.shard.receive(now, iface, frame); } - pub(crate) fn transmit(&mut self, now: Instant, credit: usize, sink: impl FnMut(&[u8])) -> usize { + pub(crate) fn transmit(&mut self, now: Instant, credit: impl Credit, sink: impl FnMut(IfIndex, &[u8])) -> usize { self.shard.transmit(now, credit, sink) } @@ -155,34 +182,38 @@ impl Stack { } /// Down, the answers go and a held lease stays: the link may come back on another network - /// that the lease's verification never hears from. - pub(crate) fn link(&mut self, now: Instant, up: bool) { - agreed(if up { self.shard.link_up(now) } else { self.shard.link_down(now) }); - if !up { - self.network.answers().empty(); + /// that the lease's verification never hears from. Up or down, a change of the link asked + /// empties them too. + pub(crate) fn link(&mut self, now: Instant, iface: IfIndex, up: bool) { + let asked = self.asked(); + agreed(if up { self.shard.link_up(now, iface) } else { self.shard.link_down(now, iface) }); + self.link_mut(iface).up = up; + if !up || self.asked() != asked { + self.answers.empty(); } } - /// What the held lease's resolvers answered, kept by the view that asked. + /// What the asked link's resolvers answered, kept by the view that asked. pub(crate) fn answers(&mut self) -> &mut Cache { - self.network.answers() + &mut self.answers } - // ---- the DHCP client's socket ---- + // ---- the DHCP clients' socket ---- - /// Queues a message of the client's; a refusal is [udp]'s, by the rule it counted. - pub(crate) fn send(&mut self, now: Instant, message: &Transmission) -> Result<(), toyos_net_udp::Error> { - self.shard.send_from(now, self.socket, message.source, message.destination, SERVER_PORT, &message.payload) + /// Queues a message of `iface`'s client; a refusal is [udp]'s, by the rule it counted. + pub(crate) fn send(&mut self, now: Instant, iface: IfIndex, message: &Transmission) -> Result<(), toyos_net_udp::Error> { + let origin = Origin { iface, source: message.source }; + self.shard.send_from(now, self.socket, origin, message.destination, SERVER_PORT, &message.payload) } - /// The oldest datagram that reached the client's socket, in `out`, and the address it came - /// from. - pub(crate) fn recv<'a>(&mut self, out: &'a mut [u8]) -> Option<(&'a [u8], Ipv4Addr)> { + /// The oldest datagram that reached the clients' socket, in `out`, the address it came from + /// and the link it came by. + pub(crate) fn recv<'a>(&mut self, out: &'a mut [u8]) -> Option<(&'a [u8], Ipv4Addr, IfIndex)> { let Ok(received) = self.shard.recv_from(self.socket, out) else { - unreachable!("the client's socket is never closed and never connected") + unreachable!("the clients' socket is never closed and never connected") }; let received = received?; - Some((out.get(..received.len)?, received.source)) + Some((out.get(..received.len)?, received.source, received.iface)) } // ---- datagram sockets: [udp]'s calls passed on, none of which writes the lease ---- @@ -243,15 +274,20 @@ impl Stack { self.shard.udp_set_ttl(id, unicast, multicast) } - pub(crate) fn join(&mut self, now: Instant, group: MulticastAddr) { - agreed(self.shard.join(now, group)); + /// Queues a datagram that leaves by `iface` alone. + pub(crate) fn send_via(&mut self, now: Instant, id: SocketId, iface: IfIndex, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), toyos_net_udp::Error> { + self.shard.send_via(now, id, iface, destination, port, payload) + } + + pub(crate) fn join(&mut self, now: Instant, iface: IfIndex, group: MulticastAddr) { + agreed(self.shard.join(now, iface, group)); } // ---- the lease ---- /// Takes what the shard has reported: each log line goes to `line`, ahead of the reports - /// before it since none is read against the state, and each report on the address waits its - /// turn for [`Self::report`]. + /// before it since none is read against a state, and each report on an address waits its turn + /// for [`Self::report`]. pub(crate) fn refusals(&mut self, mut line: impl FnMut(Refusal, u64)) { for event in self.shard.drain_events() { self.inbox.push_back(match event { @@ -259,84 +295,86 @@ impl Stack { line(refusal, suppressed); continue; } - Event::Verified(addr) => Fate::Verified(addr), - Event::Conflict { addr, mac } | Event::Lost { addr, mac } => Fate::Taken(addr, Some(mac)), - Event::NotVerified(addr) => Fate::Taken(addr, None), + Event::Verified { iface, addr } => (iface, Fate::Verified(addr)), + Event::Conflict { iface, addr, mac } | Event::Lost { iface, addr, mac } => (iface, Fate::Taken(addr, Some(mac))), + Event::NotVerified { iface, addr } => (iface, Fate::Taken(addr, None)), }); } } - /// The next report [`Self::refusals`] took, read against the state. - pub(crate) fn report(&mut self) -> Option { - Some(match self.inbox.pop_front()? { - Fate::Verified(addr) => match self.network.state() { - State::Probing(address) if *address == addr => Report::Verified(Verified(addr)), + /// The next report [`Self::refusals`] took, read against its link's state, and the link. + pub(crate) fn report(&mut self) -> Option<(IfIndex, Report)> { + let (iface, fate) = self.inbox.pop_front()?; + Some((iface, match fate { + Fate::Verified(addr) => match self.link_of(iface).state { + State::Probing(address) if address == addr => Report::Verified(Verified(addr)), _ => unreachable!("[ip] verified {addr}, which is not under probe"), }, Fate::Taken(addr, by) => { - self.taken(addr); + self.taken(iface, addr); by.map_or(Report::NotVerified, Report::Conflict) } - }) + })) } /// [ip] removed `addr` itself, its gateways with it. - fn taken(&mut self, addr: Ipv4Addr) { - if self.address() != Some(addr) { - unreachable!("[ip] removed {addr}, which the lease does not name"); + fn taken(&mut self, iface: IfIndex, addr: Ipv4Addr) { + if self.address(iface) != Some(addr) { + unreachable!("[ip] removed {addr}, which the link's lease does not name"); } - self.network.set(State::Unaddressed); + self.set(iface, State::Unaddressed); } - /// Nothing is held: the address goes, and [ip] withdraws the gateway with it. - pub(crate) fn release(&mut self, now: Instant) { - if let Some(address) = self.address() { - agreed(self.shard.remove_address(now, address)); + /// Nothing is held on `iface`: its address goes, and [ip] withdraws the gateway and ends the + /// TCP connections with it. + pub(crate) fn release(&mut self, now: Instant, iface: IfIndex) { + if let Some(address) = self.address(iface) { + agreed(self.shard.remove_address(now, iface, address)); } - self.network.set(State::Unaddressed); + self.set(iface, State::Unaddressed); } - /// A server acknowledged `address`: [ip] probes it, and nothing uses it yet. Refused, the - /// state is `Unaddressed`. - pub(crate) fn probe(&mut self, now: Instant, address: Ipv4Addr, prefix_len: u8) -> Result<(), toyos_net_ip::Counter> { - self.release(now); - self.shard.add_address(now, address, prefix_len)?; - self.network.set(State::Probing(address)); + /// A server acknowledged `address` on `iface`: [ip] probes it, and nothing uses it yet. + /// Refused, the link's state is `Unaddressed`. + pub(crate) fn probe(&mut self, now: Instant, iface: IfIndex, address: Ipv4Addr, prefix_len: u8) -> Result<(), toyos_net_ip::Counter> { + self.release(now, iface); + self.shard.add_address(now, iface, address, prefix_len)?; + self.set(iface, State::Probing(address)); Ok(()) } /// The verified address's lease is in use from here: [ip] holds the address at the prefix /// `probe` gave it, and the router is written with the lease. Returns whether [ip] refused the /// router. - pub(crate) fn hold(&mut self, now: Instant, verified: Verified, mut lease: Lease) -> bool { + pub(crate) fn hold(&mut self, now: Instant, iface: IfIndex, verified: Verified, mut lease: Lease) -> bool { if lease.address != verified.0 { unreachable!("the client configured {}, having probed {}", lease.address, verified.0); } - let refused = self.route(now, &mut lease); - self.network.set(State::Held(lease)); + let refused = self.route(now, iface, &mut lease); + self.set(iface, State::Held(lease)); refused } /// The held address acknowledged again, with new times or a new prefix, router or resolvers. /// Returns whether [ip] refused its router. A prefix [ip] refuses changes nothing: the lease /// stays as it was held. - pub(crate) fn renew(&mut self, now: Instant, mut lease: Lease) -> Result { - if !matches!(self.network.state(), State::Held(held) if held.address == lease.address) { + pub(crate) fn renew(&mut self, now: Instant, iface: IfIndex, mut lease: Lease) -> Result { + if !matches!(&self.link_of(iface).state, State::Held(held) if held.address == lease.address) { unreachable!("the client renewed {}, which is not held", lease.address); } - self.shard.add_address(now, lease.address, lease.prefix_len)?; - let refused = self.route(now, &mut lease); - self.network.set(State::Held(lease)); + self.shard.add_address(now, iface, lease.address, lease.prefix_len)?; + let refused = self.route(now, iface, &mut lease); + self.set(iface, State::Held(lease)); Ok(refused) } /// Writes `lease`'s router, which [ip] takes only on a usable prefix. One [ip] refuses is no /// router: the lease is left without one, and so is [ip]. Returns whether it was refused. - fn route(&mut self, now: Instant, lease: &mut Lease) -> bool { - let refused = self.shard.set_gateways(now, lease.router.as_slice()).is_err(); + fn route(&mut self, now: Instant, iface: IfIndex, lease: &mut Lease) -> bool { + let refused = self.shard.set_gateways(now, iface, lease.router.as_slice()).is_err(); if refused { // [ip] kept the list it had. - agreed(self.shard.set_gateways(now, &[])); + agreed(self.shard.set_gateways(now, iface, &[])); lease.router = None; } refused @@ -368,11 +406,12 @@ mod tests { }; let Some(mac) = IndividualMac::new(MacAddr([2, 0, 0, 0, 0, 0x0a])) else { unreachable!("an individual address") }; let mut now = Instant::from_millis(3_600_000); - let Ok(mut stack) = Stack::new(now, Config { mac, receive_buffer: 65_535, send_buffer: 65_535, secrets }) else { unreachable!("a configuration [tcp] takes") }; - stack.link(now, true); - assert_eq!(stack.probe(now, A, 24), Ok(())); - while stack.shard.ip().address(stack.shard.iface(), A) == Some(toyos_net_ip::AddrState::Tentative) { - stack.transmit(now, usize::MAX, |_| {}); + let Ok(mut stack) = Stack::new(now, Config { receive_buffer: 65_535, send_buffer: 65_535, secrets }) else { unreachable!("a configuration [tcp] takes") }; + let link = stack.add_link(now, mac, Rank(0)); + stack.link(now, link, true); + assert_eq!(stack.probe(now, link, A, 24), Ok(())); + while stack.shard.ip().address(link, A) == Some(toyos_net_ip::AddrState::Tentative) { + stack.transmit(now, usize::MAX, |_, _| {}); let Some(next) = stack.next_deadline() else { unreachable!("conflict detection is still running") }; now = next; stack.fire(now); @@ -383,7 +422,7 @@ mod tests { let mut lines = Vec::new(); stack.refusals(|refusal, _| lines.push(refusal)); assert!(matches!(lines.as_slice(), [Refusal::Udp(refusal)] if refusal.rule == Counter::SendUnspecifiedDestination), "{lines:?}"); - assert!(matches!(stack.report(), Some(Report::Verified(Verified(A)))), "the verification is still to be read"); + assert!(matches!(stack.report(), Some((_, Report::Verified(Verified(A))))), "the verification is still to be read"); assert!(stack.report().is_none()); stack.refusals(|refusal, _| lines.push(refusal)); assert_eq!((lines.len(), stack.report().is_none()), (1, true), "each was handed over once"); diff --git a/system/netstack/node/src/lib.rs b/system/netstack/node/src/lib.rs index 417d1e39e0b..ce7a7c464de 100644 --- a/system/netstack/node/src/lib.rs +++ b/system/netstack/node/src/lib.rs @@ -1,10 +1,10 @@ -//! Netstack's decisions, pure: one interface of ToyOS's own stack (`toyos-net-shard`) and the -//! DHCP client (`toyos-dhcp`) that addresses it. Frames and time come in; frames leave only inside -//! [`Node::transmit`], against the credit the device offers, so there is no frame without a -//! transmit slot. Nothing here reads a clock, draws randomness or does I/O: the caller hands in +//! Netstack's decisions, pure: ToyOS's own stack (`toyos-net-shard`) over every link a driver +//! brings, each with the DHCP client (`toyos-dhcp`) that addresses it. Frames and time come in; +//! frames leave only inside [`Node::transmit`], against the credit each link offers, so there is +//! no frame without a transmit slot. Nothing here reads a clock, draws randomness or does I/O: the caller hands in //! the time, every draw and the shard's secrets. //! -//! The node carries out what the client asks and tells it what became of its address. What the +//! The node carries out what each link's client asks and tells it what became of its address. What the //! interface holds is `lease`'s to write and nobody else's: see that module for the three rules //! its types keep. A client's TCP connection and the region it lent are `streams`', a passive open //! and its owner's wakes `listeners`'. @@ -55,7 +55,9 @@ use alloc::vec; use alloc::vec::Vec; use toyos_dhcp::{AddressRequest, Client, HostName, Lease, Output}; +pub use toyos_net_shard::{Credit, IfIndex, Rank}; use toyos_net_shard::{Config, Shard}; +use toyos_net_wire::ethernet::IndividualMac; use toyos_net_wire::Instant; pub use datagram::{Datagram, DatagramId, Refused}; @@ -76,23 +78,24 @@ toyos_net_wire::counters! { pub enum Event { /// A refusal of the stack's, and how many of its rule it stands for beyond itself. Stack { refusal: toyos_net_shard::Refusal, suppressed: u64 }, - /// A refusal of the client's, and how many of its rule it stands for beyond itself. - Dhcp { refusal: toyos_dhcp::Refusal, suppressed: u64 }, - /// What became of the machine's name. - Name(toyos_mdns::Event), + /// A refusal of a link's client's, and how many of its rule it stands for beyond itself. + Dhcp { link: IfIndex, refusal: toyos_dhcp::Refusal, suppressed: u64 }, + /// What became of the machine's name on a link. + Name { link: IfIndex, event: toyos_mdns::Event }, } pub struct Node { stack: Stack, - client: Client, - /// The responder for the machine's name, once [`Self::answer_as`] started it. + /// The name each link's client asks its network to record. + host_name: Option, + /// Each link's DHCP client, in the order the links were added. + clients: Vec<(IfIndex, Client)>, + /// The responders for the machine's name, once [`Self::answer_as`] started them. name: Option, - /// The link as [`Self::link`] last reported it. - up: bool, resolver: resolve::Resolver, counters: Counters, events: Vec, - /// The client's refusals pass through it as the stack's pass through the shard's: one line a + /// The clients' refusals pass through it as the stack's pass through the shard's: one line a /// rule in any 10 s, carrying how many were not. dhcp_log: toyos_dhcp::RefusalLog, /// Room for the largest message the client accepts. @@ -107,18 +110,14 @@ pub struct Node { } impl Node { - /// One interface, link down and unaddressed, with no place for a stream or a listener until - /// [`Self::set_places`]. The client begins here, and its first DISCOVER - /// is not sent: an exchange starts over when the link comes up ([`Self::link`]). - pub fn new(now: Instant, config: Config, host_name: Option, draw: impl FnMut() -> u32) -> Result { - let mac = config.mac; - let stack = Stack::new(now, config)?; - let (client, _unsent) = Client::start(now, mac, host_name, draw); + /// No link, and no place for a stream or a listener until [`Self::set_places`]. It serves + /// from here: what needs a link is refused for want of a route until one comes. + pub fn new(now: Instant, config: Config, host_name: Option) -> Result { Ok(Self { - stack, - client, + stack: Stack::new(now, config)?, + host_name, + clients: Vec::new(), name: None, - up: false, resolver: resolve::Resolver::new(), counters: Counters::default(), events: Vec::new(), @@ -131,6 +130,19 @@ impl Node { }) } + /// A link, down and unaddressed, ranked against the others by `rank`: its client begins here, + /// and its first DISCOVER is not sent: an exchange starts when the link comes up + /// ([`Self::link`]). The machine's name, once answered, is served on it too. + pub fn add_link(&mut self, now: Instant, mac: IndividualMac, rank: Rank, draw: impl FnMut() -> u32) -> IfIndex { + let iface = self.stack.add_link(now, mac, rank); + let (client, _unsent) = Client::start(now, mac, self.host_name.clone(), draw); + self.clients.push((iface, client)); + if let Some(name) = &mut self.name { + name.add_link(now, &mut self.stack, iface); + } + iface + } + pub fn shard(&self) -> &Shard { self.stack.shard() } @@ -139,10 +151,20 @@ impl Node { &self.counters } - /// The lease in use: its address verified, its router installed, its resolvers the ones to - /// ask. `None` from the moment any of them goes. - pub fn lease(&self) -> Option<&Lease> { - self.stack.lease() + /// Every link, in the order added. + pub fn links(&self) -> impl Iterator + '_ { + self.stack.links() + } + + /// `link`'s lease in use: its address verified, its router installed. `None` from the moment + /// either goes. + pub fn lease(&self, link: IfIndex) -> Option<&Lease> { + self.stack.lease(link) + } + + /// The link whose lease's resolvers lookups ask. + pub fn asked(&self) -> Option { + self.stack.asked() } /// Log lines since the last call. @@ -150,11 +172,18 @@ impl Node { self.events.drain(..) } - // ---- the device ---- + fn client(&mut self, link: IfIndex) -> &mut Client { + match self.clients.iter_mut().find(|(at, _)| *at == link) { + Some((_, client)) => client, + None => unreachable!("{link:?} is no link of the node's"), + } + } + + // ---- the devices ---- - /// The frames the device received since the last call, one by one and at most `budget`: - /// `next` hands its next frame to the sink it is given and answers `false` once it had none. - /// One pass over the streams follows the last frame, so what the batch brought reaches each + /// The frames `link` received since the last call, one by one and at most `budget`: `next` + /// hands its next frame to the sink it is given and answers `false` once it had none. One + /// pass over the streams follows the last frame, so what the batch brought reaches each /// client in one write and not one a frame. `true` is a batch that stopped at the budget, and /// another is owed without waiting for the device to say so. /// @@ -162,14 +191,14 @@ impl Node { /// batch that waited for it to empty would keep the deadlines, the transmit opportunity and /// every stream's pass waiting with it, while a bound below what one interrupt's wait can /// bring would owe a second pass for every long wait. - pub fn receive(&mut self, now: Instant, budget: usize, mut next: impl FnMut(&mut dyn FnMut(&[u8])) -> bool, mut draw: impl FnMut() -> u32) -> bool { + pub fn receive(&mut self, now: Instant, link: IfIndex, budget: usize, mut next: impl FnMut(&mut dyn FnMut(&[u8])) -> bool, mut draw: impl FnMut() -> u32) -> bool { let mut taken = 0usize; let owed = loop { if taken == budget { break true; } if !next(&mut |frame| { - self.stack.receive(now, frame); + self.stack.receive(now, link, frame); self.settle(now, &mut draw); }) { break false; @@ -180,17 +209,18 @@ impl Node { owed } - /// A transmit opportunity with room for `credit` frames, each handed to `sink` as it is built. - /// Returns how many left. A lookup's query that found no way out as its turn came is read - /// here, and the query asked in its place leaves in this opportunity while credit lasts. - pub fn transmit(&mut self, now: Instant, credit: usize, mut sink: impl FnMut(&[u8]), mut draw: impl FnMut() -> u32) -> usize { + /// A transmit opportunity with `credit`'s room on each link, each frame handed to `sink` as + /// it is built, with the link it leaves by. Returns how many left. A lookup's query that found + /// no way out as its turn came is read here, and the query asked in its place leaves in this + /// opportunity while credit lasts. + pub fn transmit(&mut self, now: Instant, mut credit: impl Credit, mut sink: impl FnMut(IfIndex, &[u8]), mut draw: impl FnMut() -> u32) -> usize { let mut sent = 0usize; loop { - sent = sent.saturating_add(self.stack.transmit(now, credit.saturating_sub(sent), &mut sink)); + sent = sent.saturating_add(self.stack.transmit(now, &mut credit, &mut sink)); // A datagram [ip] refused as it left is a line of this opportunity. self.log(now); let asked = self.resolver.pass(now, &mut self.stack, &mut self.counters, &mut draw); - if !asked || sent >= credit { + if !asked || !self.stack.links().any(|link| credit.room(link)) { break; } } @@ -198,15 +228,14 @@ impl Node { sent } - /// The link came up or went down; the caller reports a change, not a state. Down, a held - /// lease stays and the machine's name has no link; up, the client verifies a lease it holds - /// and otherwise starts over, and the name of a lease that stayed is probed for again. - pub fn link(&mut self, now: Instant, up: bool, mut draw: impl FnMut() -> u32) { - self.stack.link(now, up); - self.up = up; + /// `link` came up or went down; the caller reports a change, not a state. Down, a held lease + /// stays and the machine's name has no link there; up, the link's client verifies a lease it + /// holds and otherwise starts over, and the name of a lease that stayed is probed for again. + pub fn link(&mut self, now: Instant, link: IfIndex, up: bool, mut draw: impl FnMut() -> u32) { + self.stack.link(now, link, up); if up { - let out = self.client.link_up(now, &mut draw); - self.carry_out(now, out, None, &mut draw); + let out = self.client(link).link_up(now, &mut draw); + self.carry_out(now, link, out, None, &mut draw); } self.settle(now, &mut draw); } @@ -215,65 +244,74 @@ impl Node { pub fn next_deadline(&self) -> Option { let name = self.name.as_ref().and_then(name::Name::next_deadline); - self.stack.next_deadline().into_iter().chain(self.client.next_deadline()).chain(name).chain(self.resolver.next_deadline()).chain(self.streams.next_deadline()).min() + let clients = self.clients.iter().filter_map(|(_, client)| client.next_deadline()); + self.stack.next_deadline().into_iter().chain(clients).chain(name).chain(self.resolver.next_deadline()).chain(self.streams.next_deadline()).min() } /// Every deadline at or before `now`; the frames they make due wait for [`Self::transmit`]. pub fn fire(&mut self, now: Instant, mut draw: impl FnMut() -> u32) { self.stack.fire(now); self.settle(now, &mut draw); - let out = self.client.timer(now, &mut draw); - self.carry_out(now, out, None, &mut draw); + for at in 0..self.clients.len() { + let Some((link, client)) = self.clients.get_mut(at) else { continue }; + let link = *link; + let out = client.timer(now, &mut draw); + self.carry_out(now, link, out, None, &mut draw); + } self.settle(now, &mut draw); self.pass(now, false); } - /// Hands the client what the shard reported and what reached its socket, and carries out - /// what it answers, until neither has more; then the name is served and the lookups are - /// carried on, against the lease as that left it. + /// Hands each link's client what the shard reported and what reached its socket, and carries + /// out what it answers, until neither has more; then the name is served and the lookups are + /// carried on, against the leases as that left them. fn settle(&mut self, now: Instant, draw: &mut impl FnMut() -> u32) { loop { self.log(now); - let (out, verified) = if let Some(report) = self.stack.report() { + let (link, out, verified) = if let Some((link, report)) = self.stack.report() { + let client = self.client(link); match report { - Report::Verified(verified) => (self.client.verified(now, &mut *draw), Some(verified)), - Report::Conflict(mac) => (self.client.conflict(now, mac, &mut *draw), None), - Report::NotVerified => (self.client.not_verified(now, &mut *draw), None), + Report::Verified(verified) => (link, client.verified(now, &mut *draw), Some(verified)), + Report::Conflict(mac) => (link, client.conflict(now, mac, &mut *draw), None), + Report::NotVerified => (link, client.not_verified(now, &mut *draw), None), } - } else if let Some((payload, from)) = self.stack.recv(&mut self.datagram) { - (self.client.receive(now, payload, from, &mut *draw), None) + } else if let Some((payload, from, link)) = self.stack.recv(&mut self.datagram) { + let Some((_, client)) = self.clients.iter_mut().find(|(at, _)| *at == link) else { unreachable!("[ip] delivered from no link of the node's") }; + (link, client.receive(now, payload, from, &mut *draw), None) } else { break; }; - self.carry_out(now, out, verified, draw); + self.carry_out(now, link, out, verified, draw); } self.serve_name(now, draw); self.resolver.pass(now, &mut self.stack, &mut self.counters, draw); } - /// The stack's and the client's log lines so far, into [`Self::drain_events`]: the one way a - /// line leaves either. + /// The stack's and the clients' log lines so far, into [`Self::drain_events`]: the one way a + /// line leaves any of them. fn log(&mut self, now: Instant) { let events = &mut self.events; self.stack.refusals(|refusal, suppressed| events.push(Event::Stack { refusal, suppressed })); - for refusal in self.client.drain_refusals() { - if let Some(suppressed) = self.dhcp_log.admit(now, refusal.rule) { - events.push(Event::Dhcp { refusal, suppressed }); + for (link, client) in &mut self.clients { + for refusal in client.drain_refusals() { + if let Some(suppressed) = self.dhcp_log.admit(now, refusal.rule) { + events.push(Event::Dhcp { link: *link, refusal, suppressed }); + } } } } - /// What one call of the client's asked for: the lease first, so a message leaves from the + /// What one call of `link`'s client asked for: the lease first, so a message leaves from the /// state its call left behind. - fn carry_out(&mut self, now: Instant, out: Output, verified: Option, draw: &mut impl FnMut() -> u32) { + fn carry_out(&mut self, now: Instant, link: IfIndex, out: Output, verified: Option, draw: &mut impl FnMut() -> u32) { use toyos_dhcp::Config::{Configured, Deconfigured, Extended, Reconfigured}; let installed = match (out.config, verified) { - (Some(Configured(lease)), Some(verified)) => Some(Ok(self.stack.hold(now, verified, lease))), + (Some(Configured(lease)), Some(verified)) => Some(Ok(self.stack.hold(now, link, verified, lease))), (Some(Configured(lease)), None) => unreachable!("the client configured {} unverified", lease.address), - (Some(Extended(lease) | Reconfigured(lease)), _) => Some(self.stack.renew(now, lease)), + (Some(Extended(lease) | Reconfigured(lease)), _) => Some(self.stack.renew(now, link, lease)), // Verified, and the client has no use for it: its lease ran out under the probe. (Some(Deconfigured(_)), _) | (None, Some(_)) => { - self.stack.release(now); + self.stack.release(now, link); None } (None, None) => None, @@ -286,19 +324,19 @@ impl Node { let mut transmit = out.transmit; match out.address { Some(AddressRequest::Probe { address, prefix_len }) => { - if self.stack.probe(now, address, prefix_len).is_err() { + if self.stack.probe(now, link, address, prefix_len).is_err() { self.counters.add(Counter::AddressRefused, 1); // An address [ip] will not probe is one the client never hears about again: // its exchange starts over. - transmit = self.client.not_verified(now, &mut *draw).transmit; + transmit = self.client(link).not_verified(now, &mut *draw).transmit; } } // Only of a link reported up twice: one that drops under a probe is `NotVerified` first. - Some(AddressRequest::Cancel(_)) => self.stack.release(now), + Some(AddressRequest::Cancel(_)) => self.stack.release(now, link), None => {} } if let Some(message) = transmit { - if self.stack.send(now, &message).is_err() { + if self.stack.send(now, link, &message).is_err() { self.counters.add(Counter::DhcpUnsent, 1); } } diff --git a/system/netstack/node/src/name.rs b/system/netstack/node/src/name.rs index 9b06bd50a47..7f2e33c92e8 100644 --- a/system/netstack/node/src/name.rs +++ b/system/netstack/node/src/name.rs @@ -1,9 +1,12 @@ -//! This machine's name on its link: `.local` answers with the held lease's address -//! (`toyos-mdns`, RFC 6762). Every decision about a message, and every byte of a probe or an -//! answer, is `toyos_mdns::Responder`'s; this is its socket on the group's port, its membership -//! of the group, its clock and its draws. +//! This machine's name on each of its links: `.local` answers with that link's held +//! lease's address (`toyos-mdns`, RFC 6762). Every decision about a message, and every byte of a +//! probe or an answer, is one link's `toyos_mdns::Responder`'s; this is their one socket on the +//! group's port, their membership of the group on each link, their clock and their draws. //! -//! - The record is the held lease's on a link that is up: its address and prefix are what the +//! - **One name, a responder per link.** A message is its arrival link's responder's alone, and +//! what a responder sends leaves by its link alone. A name lost on one link is that link's: the +//! responder there probes for the same name again later (§8.1), and the others keep it. +//! - The record is the link's held lease's on a link that is up: its address and prefix are what the //! responder is told, and it is told it has no link from the moment the lease goes or the link //! does. Nothing is probed for, announced or answered for an address still under probe, or on a //! link that is down. @@ -24,11 +27,13 @@ //! source for one on this link, which is the lease's prefix or 169.254/16, and [udp] accepted it //! as one. +use alloc::collections::BTreeMap; use alloc::vec; use alloc::vec::Vec; use core::net::Ipv4Addr; use toyos_mdns::{Host, Link, Responder, Source, To, GROUP, PORT}; +use toyos_net_shard::IfIndex; use toyos_net_udp::SocketId; use toyos_net_wire::ipv4::{MulticastAddr, Ttl}; use toyos_net_wire::{Instant, Port}; @@ -37,8 +42,10 @@ use crate::lease::Stack; use crate::{Event, Node}; pub(crate) struct Name { + host: Host<'static>, socket: SocketId, - record: Responder<'static>, + /// Each link's record, by the link. + records: BTreeMap>, /// Room for the largest datagram [udp] delivers. message: Vec, } @@ -48,78 +55,93 @@ pub(crate) fn millis(now: Instant) -> u64 { u64::try_from(now.since(Instant::from_millis(0)).as_millis()).unwrap_or(u64::MAX) } -/// Queues one message of the responder's. Returns how many [udp] refused: one or none. -fn send(stack: &mut Stack, now: Instant, socket: SocketId, to: Ipv4Addr, port: u16, message: &[u8]) -> u64 { - u64::from(stack.send_to(now, socket, to, port, message).is_err()) +fn group() -> MulticastAddr { + let Some(group) = MulticastAddr::new(Ipv4Addr::from(GROUP)) else { unreachable!("RFC 6762 §3's group is a group") }; + group } impl Name { - /// Binds the group's port, sets the TTL of what leaves it and joins the group. Refused, - /// nothing was joined. + /// Binds the group's port and sets the TTL of what leaves it, then joins the group on every + /// link. Refused, nothing was joined. fn new(now: Instant, stack: &mut Stack, host: Host<'static>) -> Result { - let (Ok(ttl), Some(group)) = (Ttl::new(u8::MAX), MulticastAddr::new(Ipv4Addr::from(GROUP))) else { - unreachable!("255 is a TTL and RFC 6762 §3's group is a group") - }; + let Ok(ttl) = Ttl::new(u8::MAX) else { unreachable!("255 is a TTL") }; let (socket, _) = stack.bind(Ipv4Addr::UNSPECIFIED, Port::new(PORT), || 0)?; stack.set_ttl(socket, ttl, ttl)?; - stack.join(now, group); - Ok(Self { socket, record: Responder::new(host), message: vec![0; toyos_net_udp::limits::MAX_PAYLOAD] }) + let mut name = Self { host, socket, records: BTreeMap::new(), message: vec![0; toyos_net_udp::limits::MAX_PAYLOAD] }; + let links: Vec = stack.links().collect(); + for link in links { + name.add_link(now, stack, link); + } + Ok(name) + } + + /// Joins the group on `link`, whose record begins with no address. + pub(crate) fn add_link(&mut self, now: Instant, stack: &mut Stack, link: IfIndex) { + stack.join(now, link, group()); + self.records.insert(link, Responder::new(self.host)); } pub(crate) fn next_deadline(&self) -> Option { - self.record.owed_at().map(Instant::from_millis) + self.records.values().filter_map(Responder::owed_at).min().map(Instant::from_millis) } - /// Tells the responder what lease is held and whether the link is `up`, hands it every - /// message that arrived and sends what each is answered, then sends what the name is owed - /// now. What became of the name goes to `events`. Returns how many messages [udp] refused. - fn pass(&mut self, now: Instant, stack: &mut Stack, up: bool, events: &mut Vec, draw: &mut impl FnMut() -> u32) -> u64 { - let link = stack.lease().filter(|_| up).map(|lease| Link { addr: lease.address.octets(), prefix: lease.prefix_len }); + /// Tells each link's responder what lease is held there and whether the link is up, hands + /// each message that arrived to its link's responder and sends what it is answered by that + /// link, then sends what each link's name is owed now. What became of a name goes to + /// `events`. Returns how many messages [udp] refused. + fn pass(&mut self, now: Instant, stack: &mut Stack, events: &mut Vec, draw: &mut impl FnMut() -> u32) -> u64 { let (ms, group) = (millis(now), Ipv4Addr::from(GROUP)); let mut unsent = 0u64; - self.record.on(link, ms); + for (&link, record) in &mut self.records { + let on = stack.lease(link).filter(|_| stack.up(link)).map(|lease| Link { addr: lease.address.octets(), prefix: lease.prefix_len }); + record.on(on, ms); + } + let socket = self.socket; + let send = |stack: &mut Stack, link: IfIndex, to: Ipv4Addr, port: u16, message: &[u8]| u64::from(stack.send_via(now, socket, link, to, port, message).is_err()); loop { let received = match stack.recv_from(self.socket, &mut self.message) { Ok(Some(received)) => received, Ok(None) => break, - Err(_) => unreachable!("the responder's socket is never closed and never connected"), + Err(_) => unreachable!("the responders' socket is never closed and never connected"), }; - let Some(message) = self.message.get(..received.len) else { continue }; + let (Some(message), Some(record)) = (self.message.get(..received.len), self.records.get_mut(&received.iface)) else { continue }; // Port 0 is no port to answer at: [udp] refuses the answer. let port = received.source_port.map_or(0, Port::get); - let (answer, event) = self.record.heard(message, Source { addr: received.source.octets(), port }, ms); - events.extend(event.map(Event::Name)); + let (answer, event) = record.heard(message, Source { addr: received.source.octets(), port }, ms); + events.extend(event.map(|event| Event::Name { link: received.iface, event })); let Some(answer) = answer else { continue }; let (to, port) = match answer.to { To::Group => (group, PORT), To::Asker => (received.source, port), }; - unsent = unsent.saturating_add(send(stack, now, self.socket, to, port, &answer.bytes)); + unsent = unsent.saturating_add(send(stack, received.iface, to, port, &answer.bytes)); } - let (owed, event) = self.record.owed(ms, draw); - events.extend(event.map(Event::Name)); - if let Some(owed) = owed { - unsent = unsent.saturating_add(send(stack, now, self.socket, group, PORT, &owed)); + for (&link, record) in &mut self.records { + let (owed, event) = record.owed(ms, &mut *draw); + events.extend(event.map(|event| Event::Name { link, event })); + if let Some(owed) = owed { + unsent = unsent.saturating_add(send(stack, link, group, PORT, &owed)); + } } unsent } } impl Node { - /// Answers for `.local` from here on, once it is claimed: the node joins the multicast - /// DNS group and holds its port, and a lease already held on a link that is up is probed on - /// by this call, which then spends one draw. Refused when the port is taken, by a client's - /// socket or by an earlier call. + /// Answers for `.local` from here on, on every link, once it is claimed there: the node + /// joins the multicast DNS group on each and holds its port, and a lease already held on a + /// link that is up is probed on by this call, which then spends one draw for each. Refused + /// when the port is taken, by a client's socket or by an earlier call. pub fn answer_as(&mut self, now: Instant, host: Host<'static>, mut draw: impl FnMut() -> u32) -> Result<(), toyos_net_udp::Error> { self.name = Some(Name::new(now, &mut self.stack, host)?); self.serve_name(now, &mut draw); Ok(()) } - /// The name's pass, if it has been started, and the count of what [udp] refused it. + /// The names' pass, if they have been started, and the count of what [udp] refused them. pub(crate) fn serve_name(&mut self, now: Instant, draw: &mut impl FnMut() -> u32) { if let Some(name) = &mut self.name { - let unsent = name.pass(now, &mut self.stack, self.up, &mut self.events, draw); + let unsent = name.pass(now, &mut self.stack, &mut self.events, draw); self.counters.add(crate::Counter::NameUnsent, unsent); } } diff --git a/system/netstack/node/src/resolve.rs b/system/netstack/node/src/resolve.rs index 917fa53147c..f3fb329cee8 100644 --- a/system/netstack/node/src/resolve.rs +++ b/system/netstack/node/src/resolve.rs @@ -207,7 +207,7 @@ impl Resolver { if let Some(kept) = stack.answers().answer(view, &name, millis(now)) { return Ok(Resolving::Kept(kept.map_err(Ended::Failed))); } - let resolvers = stack.lease().map(|lease| lease.dns.clone()).unwrap_or_default(); + let resolvers = stack.asked_lease().map(|lease| lease.dns.clone()).unwrap_or_default(); if resolvers.is_empty() { return Err(NotStarted::NotConnected); } @@ -242,7 +242,7 @@ impl Resolver { let ms = millis(now); let mut queued = false; lookups.retain_mut(|asking| { - let named = stack.lease().is_some_and(|lease| lease.dns == asking.resolvers); + let named = stack.asked_lease().is_some_and(|lease| lease.dns == asking.resolvers); let mut done = if named { None } else { Some(Err(Ended::LeaseChanged)) }; while done.is_none() { let Some((query, heard)) = waiting(&asking.queries, stack, reply.as_mut_slice()) else { break }; diff --git a/system/netstack/node/src/streams.rs b/system/netstack/node/src/streams.rs index 067ec99ac0c..f7e3e88d4d2 100644 --- a/system/netstack/node/src/streams.rs +++ b/system/netstack/node/src/streams.rs @@ -29,7 +29,7 @@ use alloc::collections::{BTreeMap, VecDeque}; use core::sync::atomic::Ordering; use core::time::Duration; -use toyos::net::stream::{Server, Words, LAYOUT, NODELAY, RECEIVE_BYTES, RESET, TIMED_OUT, UNREACHABLE}; +use toyos::net::stream::{Server, Words, ADDRESS_GONE, LAYOUT, NODELAY, RECEIVE_BYTES, RESET, TIMED_OUT, UNREACHABLE}; use toyos_net_shard::ConnectError; use toyos_net_tcp::{ConnId, Endpoint, Error, Failure, Lent, Options, State, Status}; use toyos_net_wire::{Instant, Port}; @@ -92,6 +92,7 @@ const fn code(failure: Failure) -> u32 { Failure::TimedOut => TIMED_OUT, Failure::Unreachable(_) | Failure::Prohibited => UNREACHABLE, Failure::Refused | Failure::Reset => RESET, + Failure::AddressGone => ADDRESS_GONE, } } diff --git a/system/netstack/node/tests/common/mod.rs b/system/netstack/node/tests/common/mod.rs index 862d1bbd0d5..36f54fad9dd 100644 --- a/system/netstack/node/tests/common/mod.rs +++ b/system/netstack/node/tests/common/mod.rs @@ -18,7 +18,7 @@ use std::time::Duration; use etherparse::{ArpOperation, Icmpv4Type, LinkSlice, NetSlice, SlicedPacket, TransportSlice}; use toyos_dhcp::HostName; -use toyos_net_node::{Bell, Node}; +use toyos_net_node::{Bell, IfIndex, Node, Rank}; use toyos_net_shard::{Config, Secrets}; use toyos_net_wire::ethernet::{IndividualMac, MacAddr}; use toyos_net_wire::Instant; @@ -329,6 +329,8 @@ pub fn batch<'a>(frames: &'a [&'a [u8]]) -> impl FnMut(&mut Sink<'_>) -> bool + pub struct Wire { pub node: Node, + /// The node's one link. + pub link: IfIndex, mac: [u8; 6], pub now: Instant, /// Every frame the node emitted, in order. @@ -358,11 +360,12 @@ impl Wire { tcp: toyos_net_tcp::Secrets { isn: [3; 16], timestamp: [4; 16], port_offset: [5; 16], port_index: [6; 16], port_table: [0; 16] }, }; let now = Instant::from_millis(3_600_000); - let config = Config { mac: IndividualMac::new(MacAddr(mac)).unwrap(), receive_buffer: 65_535, send_buffer: 65_535, secrets }; + let config = Config { receive_buffer: 65_535, send_buffer: 65_535, secrets }; let mut draws = 0x5a00_0000; - let mut node = Node::new(now, config, HostName::new("toyos"), draw(&mut draws)).unwrap(); + let mut node = Node::new(now, config, HostName::new("toyos")).unwrap(); + let link = node.add_link(now, IndividualMac::new(MacAddr(mac)).unwrap(), Rank(0), draw(&mut draws)); node.set_places(PLACES); - Self { node, mac, now, sent: Vec::new(), draws } + Self { node, link, mac, now, sent: Vec::new(), draws } } /// Offers the node all the credit it wants until it sends nothing more, the router answering @@ -371,13 +374,13 @@ impl Wire { loop { let from = self.sent.len(); let (sent, mac) = (&mut self.sent, self.mac); - self.node.transmit(self.now, usize::MAX, |frame| sent.push(outside(frame, mac)), draw(&mut self.draws)); + self.node.transmit(self.now, usize::MAX, |_, frame| sent.push(outside(frame, mac)), draw(&mut self.draws)); if self.sent.len() == from { return; } let asked = self.sent[from..].iter().filter(|seen| matches!(seen, Seen::Arp { request: true, sender, target, .. } if *target == R && *sender == A)).count(); for _ in 0..asked { - self.node.receive(self.now, BUDGET, batch(&[&arp(MAC, false, MAC_R, R, A)]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[&arp(MAC, false, MAC_R, R, A)]), draw(&mut self.draws)); } } } @@ -388,12 +391,12 @@ impl Wire { } pub fn deliver(&mut self, frame: &[u8]) { - self.node.receive(self.now, BUDGET, batch(&[frame]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[frame]), draw(&mut self.draws)); self.pump(); } pub fn link(&mut self, up: bool) { - self.node.link(self.now, up, draw(&mut self.draws)); + self.node.link(self.now, self.link, up, draw(&mut self.draws)); self.pump(); } @@ -434,13 +437,11 @@ impl Wire { /// The interface's address `address`, in [ip]'s words. pub fn address(&self, address: Ipv4Addr) -> Option { - let shard = self.node.shard(); - shard.ip().address(shard.iface(), address) + self.node.shard().ip().address(self.link, address) } pub fn gateways(&self) -> Vec { - let shard = self.node.shard(); - shard.ip().gateways(shard.iface()).expect("the interface").to_vec() + self.node.shard().ip().gateways(self.link).expect("the interface").to_vec() } /// Whether the node answers an ARP request for `address` from the router. @@ -465,7 +466,7 @@ impl Wire { /// [`Self::acknowledged`], and conflict detection run to its end: the lease is held. pub fn leased(terms: &[(u8, Vec)]) -> Self { let mut wire = Self::acknowledged(terms); - assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease().is_some()), "the lease is held"); + assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease(wire.link).is_some()), "the lease is held"); wire } } diff --git a/system/netstack/node/tests/datagram.rs b/system/netstack/node/tests/datagram.rs index d61de97c63f..ae37485013b 100644 --- a/system/netstack/node/tests/datagram.rs +++ b/system/netstack/node/tests/datagram.rs @@ -144,7 +144,7 @@ fn each_refusal_is_answered_in_the_pipes_word_for_it() { lan.pump(); lan.node.udp_send_to(lan.now, any, B, 7, b"later").expect("the same call, once the queue has left"); - assert!(lan.run_until(Duration::from_secs(700), |lan| lan.node.lease().is_none()), "the lease runs out"); + assert!(lan.run_until(Duration::from_secs(700), |lan| lan.node.lease(lan.link).is_none()), "the lease runs out"); refused_send(&mut lan, any, B, 7, b"late", Refused::NotConnected, Counter::NoSourceAddress); refused_send(&mut lan, held, B, 7, b"late", Refused::NotConnected, Counter::SourceAddressNotAssigned); @@ -218,13 +218,13 @@ fn a_datagram_accepted_for_a_host_is_no_broadcast_after_the_server_renews_a_narr lan.node.udp_send_to(lan.now, id, edge, 5_001, b"hi").expect("one host of the /24"); let narrower = [(54, R.octets().to_vec()), (51, 600u32.to_be_bytes().to_vec()), (1, vec![255, 255, 255, 128]), (6, DNS.octets().to_vec())]; lan.deliver(&from_server(MAC, A, &message_of(ACK, renewal, &narrower))); - assert_eq!(lan.node.lease().map(|lease| lease.prefix_len), Some(25), "the server's prefix is the lease's"); + assert_eq!(lan.node.lease(lan.link).map(|lease| lease.prefix_len), Some(25), "the server's prefix is the lease's"); assert_eq!(lan.sent[frames..], [], "nothing left for it: no broadcast, and no request for a link address"); let rule = toyos_net_ip::Counter::IpBroadcastNotPermitted; let shard = lan.node.shard(); assert_eq!(shard.ip().counters().get(rule), 1); - let refusal = toyos_net_ip::Refusal { rule, iface: shard.iface(), peer: toyos_net_ip::Peer::Ip(edge) }; + let refusal = toyos_net_ip::Refusal { rule, iface: lan.link, peer: toyos_net_ip::Peer::Ip(edge) }; let logged: Vec = lan.node.drain_events().collect(); assert_eq!(logged, [Event::Stack { refusal: Refusal::Ip(refusal), suppressed: 0 }]); } diff --git a/system/netstack/node/tests/lan/mod.rs b/system/netstack/node/tests/lan/mod.rs index de4ec988d58..bb9e52def2d 100644 --- a/system/netstack/node/tests/lan/mod.rs +++ b/system/netstack/node/tests/lan/mod.rs @@ -117,6 +117,7 @@ pub fn udp(to: [u8; 6], from: [u8; 6], source: (Ipv4Addr, u16), destination: (Ip pub struct Lan { pub node: Node, + pub link: toyos_net_node::IfIndex, pub now: Instant, /// Every frame the node emitted, in order, with the time it left. pub sent: Vec<(Instant, Seen)>, @@ -134,14 +135,14 @@ fn draw(draws: &mut u32) -> impl FnMut() -> u32 + '_ { impl Lan { /// `common`'s node, an hour into the clock, its link down. pub fn new() -> Self { - let Wire { node, now, .. } = Wire::new(); - Self { node, now, sent: Vec::new(), draws: 0x6b00_0000 } + let Wire { node, link, now, .. } = Wire::new(); + Self { node, link, now, sent: Vec::new(), draws: 0x6b00_0000 } } /// One transmit opportunity with all the credit the node wants. Returns how many frames left. pub fn opportunity(&mut self) -> usize { let (sent, now) = (&mut self.sent, self.now); - self.node.transmit(now, usize::MAX, |frame| sent.push((now, outside(frame))), draw(&mut self.draws)) + self.node.transmit(now, usize::MAX, |_, frame| sent.push((now, outside(frame))), draw(&mut self.draws)) } /// Opportunities until the node sends nothing more, the router and B answering each ARP @@ -161,18 +162,18 @@ impl Lan { .collect(); for target in asked { let mac = if target == R { MAC_R } else { MAC_B }; - self.node.receive(self.now, BUDGET, batch(&[&arp(MAC, false, mac, target, A)]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[&arp(MAC, false, mac, target, A)]), draw(&mut self.draws)); } } } pub fn deliver(&mut self, frame: &[u8]) { - self.node.receive(self.now, BUDGET, batch(&[frame]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[frame]), draw(&mut self.draws)); self.pump(); } pub fn link(&mut self, up: bool) { - self.node.link(self.now, up, draw(&mut self.draws)); + self.node.link(self.now, self.link, up, draw(&mut self.draws)); self.pump(); } @@ -208,13 +209,13 @@ impl Lan { let id = xid(&discover); self.deliver(&from_server(MAC, A, &message_of(OFFER, id, terms))); self.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, id, terms))); - assert_eq!(self.node.lease(), None, "acknowledged is not held"); + assert_eq!(self.node.lease(self.link), None, "acknowledged is not held"); } /// [`Self::acknowledge`], and conflict detection run to its end: the lease is held. pub fn lease_on(&mut self, terms: &[(u8, Vec)]) { self.acknowledge(terms); - assert!(self.run_until(Duration::from_secs(10), |lan| lan.node.lease().is_some()), "the lease is held"); + assert!(self.run_until(Duration::from_secs(10), |lan| lan.node.lease(lan.link).is_some()), "the lease is held"); } /// A lease of `A`/24 for `seconds`, through the router. diff --git a/system/netstack/node/tests/lease.rs b/system/netstack/node/tests/lease.rs index 460a578bb3f..b35f99f7b3e 100644 --- a/system/netstack/node/tests/lease.rs +++ b/system/netstack/node/tests/lease.rs @@ -56,15 +56,15 @@ fn the_lease_end_to_end() { let acknowledged = wire.sent.len(); wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, id, &terms))); - assert_eq!((wire.node.lease(), wire.address(A)), (None, Some(AddrState::Tentative))); + assert_eq!((wire.node.lease(wire.link), wire.address(A)), (None, Some(AddrState::Tentative))); - assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease().is_some()), "the lease is held"); + assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease(wire.link).is_some()), "the lease is held"); let detection = &wire.sent[acknowledged..]; assert_eq!(detection.len(), 4, "three probes and an announcement: {detection:?}"); assert!(detection[..3].iter().all(|frame| frame.is_probe(A)), "{detection:?}"); assert!(detection[3].is_announcement(A), "{detection:?}"); - let lease = wire.node.lease().unwrap(); + let lease = wire.node.lease(wire.link).unwrap(); assert_eq!((lease.address, lease.prefix_len, lease.router, lease.dns.as_slice(), lease.server), (A, 24, Some(R), &[DNS][..], R)); assert_eq!(wire.address(A), Some(AddrState::Announcing)); assert_eq!(wire.gateways(), [R]); @@ -81,9 +81,9 @@ fn the_lease_end_to_end() { #[test] fn no_frame_is_from_the_address_before_it_is_verified() { let mut wire = Wire::acknowledged(&terms(HOUR, Some(R))); - assert_eq!(wire.node.lease(), None, "acknowledged is not held"); + assert_eq!(wire.node.lease(wire.link), None, "acknowledged is not held"); for _ in 0..16 { - if wire.node.lease().is_some() { + if wire.node.lease(wire.link).is_some() { break; } assert_eq!(wire.address(A), Some(AddrState::Tentative)); @@ -92,7 +92,7 @@ fn no_frame_is_from_the_address_before_it_is_verified() { let at = wire.node.next_deadline().expect("conflict detection is running"); wire.fire(at); } - assert!(wire.node.lease().is_some(), "held within sixteen deadlines"); + assert!(wire.node.lease(wire.link).is_some(), "held within sixteen deadlines"); let probes = wire.sent.iter().filter(|frame| frame.is_probe(A)).count(); assert_eq!(probes, 3, "held only once the third probe has left"); assert!(wire.sent.iter().all(|frame| frame.source() != Some(A)), "{:?}", wire.sent); @@ -101,10 +101,10 @@ fn no_frame_is_from_the_address_before_it_is_verified() { #[test] fn expiry_takes_address_route_and_resolvers_in_one_step() { let mut wire = Wire::leased(&terms(600, Some(R))); - assert_eq!((wire.address(A).is_some(), wire.gateways(), wire.node.lease().map(|lease| lease.dns.clone())), (true, vec![R], Some(vec![DNS]))); + assert_eq!((wire.address(A).is_some(), wire.gateways(), wire.node.lease(wire.link).map(|lease| lease.dns.clone())), (true, vec![R], Some(vec![DNS]))); let held = wire.sent.len(); - assert!(wire.run_until(Duration::from_secs(700), |wire| wire.node.lease().is_none()), "the lease runs out"); + assert!(wire.run_until(Duration::from_secs(700), |wire| wire.node.lease(wire.link).is_none()), "the lease runs out"); assert_eq!(wire.now, after(600), "at its expiry"); assert_eq!((wire.address(A), wire.gateways()), (None, vec![])); let renewal = wire.sent[held..].iter().find(|frame| frame.dhcp().is_some()).expect("a renewal"); @@ -136,13 +136,13 @@ fn a_renewal_for_a_server_nobody_answers_for_waits_and_is_dropped_and_the_client assert!(wire.run_until(Duration::from_secs(10), |wire| dropped(wire) == 1), "the renewal is dropped"); assert_eq!((wire.now, asks(&wire)), (renewed.after(Duration::from_secs(3)), 3), "when [ip] gave the server up"); assert!(wire.sent[held..].iter().all(|seen| seen.dhcp().is_none()), "no renewal left"); - assert!(wire.node.lease().is_some(), "the lease is held, and nothing reached the client"); + assert!(wire.node.lease(wire.link).is_some(), "the lease is held, and nothing reached the client"); assert!(wire.run_until(Duration::from_secs(400), |wire| wire.sent[held..].iter().any(|seen| seen.dhcp().is_some())), "the client asks on"); assert!(dropped(&wire) > 1, "a renewal again first, on the client's clock"); let Some(Seen::Dhcp { to, source, destination, message }) = wire.sent.last() else { panic!("{:?}", wire.sent.last()) }; assert_eq!((*to, *source, *destination, option(message, 53)), (BROADCAST, A, Ipv4Addr::BROADCAST, Some(&[REQUEST][..])), "the rebinding request"); - assert!(wire.node.lease().is_some()); + assert!(wire.node.lease(wire.link).is_some()); } #[test] @@ -150,10 +150,10 @@ fn a_nak_takes_address_route_and_resolvers_in_one_step() { let mut wire = Wire::leased(&terms(600, Some(R))); let id = renewing(&mut wire); wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message(NAK, id, Ipv4Addr::UNSPECIFIED, &[(54, R.octets().to_vec())]))); - assert_eq!((wire.node.lease(), wire.address(A), wire.gateways()), (None, None, vec![])); + assert_eq!((wire.node.lease(wire.link), wire.address(A), wire.gateways()), (None, None, vec![])); assert!(matches!(wire.sent.last(), Some(Seen::Dhcp { source, .. }) if source.is_unspecified()), "{:?}", wire.sent.last()); assert_ne!(xid(wire.last(DISCOVER)), id, "a new exchange"); - let refused = Event::Dhcp { refusal: toyos_dhcp::Refusal { rule: toyos_dhcp::Counter::Nak, peer: toyos_dhcp::Peer::From(R) }, suppressed: 0 }; + let refused = Event::Dhcp { link: wire.link, refusal: toyos_dhcp::Refusal { rule: toyos_dhcp::Counter::Nak, peer: toyos_dhcp::Peer::From(R) }, suppressed: 0 }; assert!(wire.node.drain_events().any(|event| event == refused)); assert!(!wire.answers_arp_for(A)); } @@ -164,7 +164,7 @@ fn a_renewal_without_a_router_takes_the_route_and_keeps_the_address() { let id = renewing(&mut wire); let renewed = wire.now; wire.deliver(&from_server(MAC, A, &message_of(ACK, id, &terms(600, None)))); - let lease = wire.node.lease().expect("still held"); + let lease = wire.node.lease(wire.link).expect("still held"); assert_eq!((lease.address, lease.router, lease.timers.map(|timers| timers.expiry)), (A, None, Some(renewed.after(Duration::from_secs(600))))); assert_eq!((wire.address(A), wire.gateways()), (Some(AddrState::Assigned), vec![])); assert_eq!(wire.node.counters().get(Counter::RouterRefused), 0); @@ -178,10 +178,10 @@ fn a_renewed_prefix_reaches_ip_before_its_router() { let (wider, beyond) = (Ipv4Addr::new(255, 255, 254, 0), Ipv4Addr::new(192, 0, 3, 254)); let terms = [(54, R.octets().to_vec()), (51, 600u32.to_be_bytes().to_vec()), (1, wider.octets().to_vec()), (3, beyond.octets().to_vec()), (6, DNS.octets().to_vec())]; wire.deliver(&from_server(MAC, A, &message_of(ACK, id, &terms))); - let lease = wire.node.lease().expect("still held"); + let lease = wire.node.lease(wire.link).expect("still held"); assert_eq!((lease.prefix_len, lease.router), (23, Some(beyond))); let shard = wire.node.shard(); - assert_eq!(shard.ip().prefix_len(shard.iface(), A), Some(23)); + assert_eq!(shard.ip().prefix_len(wire.link, A), Some(23)); assert_eq!(wire.gateways(), [beyond]); assert_eq!(wire.node.counters().get(Counter::RouterRefused), 0); } @@ -194,7 +194,7 @@ fn a_lease_that_runs_out_under_the_probe_leaves_no_address() { assert!(wire.run_until(Duration::from_secs(10), |wire| wire.dhcp().len() == 3), "the client starts over"); wire.last(DISCOVER); assert_eq!(wire.sent.iter().filter(|frame| frame.is_probe(A)).count(), 3, "detection ran to its end"); - assert_eq!((wire.node.lease(), wire.address(A), wire.gateways()), (None, None, vec![])); + assert_eq!((wire.node.lease(wire.link), wire.address(A), wire.gateways()), (None, None, vec![])); assert!(!wire.answers_arp_for(A)); } @@ -207,12 +207,12 @@ fn a_second_conflict_takes_a_held_lease_and_declines_it() { let from = wire.sent.len(); wire.deliver(&conflict); assert_eq!(wire.sent[from..].iter().filter(|frame| frame.is_announcement(A)).count(), 1, "defended once: {:?}", &wire.sent[from..]); - assert_eq!((wire.node.lease().map(|lease| lease.address), wire.gateways()), (Some(A), vec![R])); + assert_eq!((wire.node.lease(wire.link).map(|lease| lease.address), wire.gateways()), (Some(A), vec![R])); // Inside [ip]'s defend interval (RFC 5227 §2.4 (b)). wire.fire(after(105)); wire.deliver(&conflict); - assert_eq!((wire.node.lease(), wire.address(A), wire.gateways()), (None, None, vec![])); + assert_eq!((wire.node.lease(wire.link), wire.address(A), wire.gateways()), (None, None, vec![])); let decline = wire.last(DECLINE); assert_eq!((option(decline, 50), option(decline, 54)), (Some(&A.octets()[..]), Some(&R.octets()[..]))); assert!(!wire.answers_arp_for(A)); @@ -284,7 +284,7 @@ fn losses() -> [(&'static str, Loss); 3] { wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message(NAK, id, Ipv4Addr::UNSPECIFIED, &[(54, R.octets().to_vec())]))); } fn expiry(wire: &mut Wire) { - assert!(wire.run_until(Duration::from_secs(700), |wire| wire.node.lease().is_none()), "the lease runs out"); + assert!(wire.run_until(Duration::from_secs(700), |wire| wire.node.lease(wire.link).is_none()), "the lease runs out"); } fn conflict(wire: &mut Wire) { let conflict = arp(BROADCAST, false, MAC_B, A, A); @@ -320,11 +320,11 @@ fn a_listener_stands_while_its_address_is_lost_and_answers_when_it_is_back() { wire.link(true); let id = xid(wire.last(REQUEST)); wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, id, &terms(600, Some(R))))); - assert_eq!(wire.node.lease().map(|lease| lease.address), Some(A), "{how}"); + assert_eq!(wire.node.lease(wire.link).map(|lease| lease.address), Some(A), "{how}"); stand(&mut wire, "renewed, and the link back"); lose(&mut wire); - assert_eq!((wire.node.lease(), wire.address(A)), (None, None), "{how}"); + assert_eq!((wire.node.lease(wire.link), wire.address(A)), (None, None), "{how}"); stand(&mut wire, "the address lost"); assert_eq!(syn(&mut wire, 22), [], "{how}: a SYN to an address the machine does not hold"); stand(&mut wire, "a SYN to the lost address"); @@ -334,7 +334,7 @@ fn a_listener_stands_while_its_address_is_lost_and_answers_when_it_is_back() { let id = xid(wire.last(DISCOVER)); wire.deliver(&from_server(MAC, A, &message_of(OFFER, id, &terms(600, Some(R))))); wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, id, &terms(600, Some(R))))); - assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease().is_some()), "{how}: the address is held again"); + assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease(wire.link).is_some()), "{how}: the address is held again"); stand(&mut wire, "the address back"); let answer = syn(&mut wire, 22); @@ -349,7 +349,7 @@ fn a_listener_stands_while_its_address_is_lost_and_answers_when_it_is_back() { fn a_router_ip_refuses_is_no_router() { let edge = Ipv4Addr::new(192, 0, 2, 255); let wire = Wire::leased(&terms(HOUR, Some(edge))); - assert_eq!(wire.node.lease().map(|lease| lease.router), Some(None)); + assert_eq!(wire.node.lease(wire.link).map(|lease| lease.router), Some(None)); assert!(wire.gateways().is_empty()); assert_eq!(wire.node.counters().get(Counter::RouterRefused), 1); } @@ -366,7 +366,7 @@ fn a_conflict_under_the_probe_declines_and_holds_nothing() { let Some(Seen::Dhcp { to, source, destination, message }) = wire.sent.last() else { panic!("{:?}", wire.sent.last()) }; assert_eq!((*to, *source, *destination), (BROADCAST, Ipv4Addr::UNSPECIFIED, Ipv4Addr::BROADCAST)); assert_eq!((option(message, 53), option(message, 50), option(message, 54)), (Some(&[DECLINE][..]), Some(&A.octets()[..]), Some(&R.octets()[..]))); - assert_eq!((wire.node.lease(), wire.address(A)), (None, None)); + assert_eq!((wire.node.lease(wire.link), wire.address(A)), (None, None)); let events: Vec = wire.node.drain_events().collect(); let stack = |event: &Event| matches!(event, Event::Stack { refusal: Refusal::Ip(refusal), .. } if refusal.rule == toyos_net_ip::Counter::AcdConflict); let dhcp = |event: &Event| matches!(event, Event::Dhcp { refusal, .. } if refusal.rule == toyos_dhcp::Counter::Declined); @@ -374,7 +374,7 @@ fn a_conflict_under_the_probe_declines_and_holds_nothing() { assert!(wire.run_until(Duration::from_secs(11), |wire| wire.dhcp().last().is_some_and(|(kind, _)| *kind == DISCOVER)), "discovery starts over"); assert_eq!(wire.now, conflict.after(toyos_dhcp::limits::DECLINE_BACKOFF)); - assert_eq!(wire.node.lease(), None); + assert_eq!(wire.node.lease(wire.link), None); } #[test] @@ -382,18 +382,18 @@ fn a_link_that_returns_keeps_a_held_lease_and_announces_it() { let mut wire = Wire::leased(&terms(HOUR, Some(R))); wire.fire(after(100)); wire.link(false); - assert_eq!((wire.node.lease().map(|lease| lease.address), wire.address(A).is_some(), wire.gateways()), (Some(A), true, vec![R])); + assert_eq!((wire.node.lease(wire.link).map(|lease| lease.address), wire.address(A).is_some(), wire.gateways()), (Some(A), true, vec![R])); let down = wire.sent.len(); wire.link(true); - assert_eq!(wire.node.lease().map(|lease| lease.address), Some(A)); + assert_eq!(wire.node.lease(wire.link).map(|lease| lease.address), Some(A)); assert!(wire.sent[down..].iter().any(|frame| frame.is_announcement(A)), "{:?}", &wire.sent[down..]); let request = wire.last(REQUEST); assert_eq!((ciaddr(request), option(request, 50), option(request, 54)), (Ipv4Addr::UNSPECIFIED, Some(&A.octets()[..]), None)); let id = xid(request); wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, id, &terms(HOUR, Some(R))))); - let expiry = wire.node.lease().and_then(|lease| lease.timers).map(|timers| timers.expiry); + let expiry = wire.node.lease(wire.link).and_then(|lease| lease.timers).map(|timers| timers.expiry); assert_eq!(expiry, Some(after(100 + u64::from(HOUR))), "the lease runs from the request the server acknowledged"); assert_eq!(wire.gateways(), [R]); } @@ -415,7 +415,7 @@ fn a_probe_the_link_drops_under_is_abandoned() { let mut wire = Wire::acknowledged(&terms(HOUR, Some(R))); let sent = wire.sent.len(); wire.link(false); - assert_eq!((wire.node.lease(), wire.address(A)), (None, None)); + assert_eq!((wire.node.lease(wire.link), wire.address(A)), (None, None)); assert_eq!((wire.sent.len(), wire.node.counters().get(Counter::DhcpUnsent)), (sent, 1), "its DISCOVER has no link to leave by"); wire.link(true); wire.last(DISCOVER); @@ -468,7 +468,7 @@ fn a_reply_without_the_cookie_is_refused_by_name() { bootp[236..240].fill(0); wire.deliver(&from_server(MAC, A, &bootp)); wire.last(DISCOVER); - let refused = Event::Dhcp { refusal: toyos_dhcp::Refusal { rule: toyos_dhcp::Counter::BootpReply, peer: toyos_dhcp::Peer::From(R) }, suppressed: 0 }; + let refused = Event::Dhcp { link: wire.link, refusal: toyos_dhcp::Refusal { rule: toyos_dhcp::Counter::BootpReply, peer: toyos_dhcp::Peer::From(R) }, suppressed: 0 }; assert!(wire.node.drain_events().any(|event| event == refused)); assert_eq!(wire.node.drain_events().count(), 0, "drained"); } @@ -486,11 +486,11 @@ fn a_hundred_refused_replies_are_one_line_and_a_count() { for _ in 0..100 { wire.deliver(&from_server(MAC, A, &bootp)); } - assert_eq!(wire.node.drain_events().collect::>(), [Event::Dhcp { refusal, suppressed: 0 }]); + assert_eq!(wire.node.drain_events().collect::>(), [Event::Dhcp { link: wire.link, refusal, suppressed: 0 }]); wire.now = wire.now.after(toyos_net_wire::REFUSAL_LOG_INTERVAL); wire.deliver(&from_server(MAC, A, &bootp)); - assert_eq!(wire.node.drain_events().collect::>(), [Event::Dhcp { refusal, suppressed: 99 }]); + assert_eq!(wire.node.drain_events().collect::>(), [Event::Dhcp { link: wire.link, refusal, suppressed: 99 }]); } /// The node waiting for the ACK of its REQUEST, and that ACK's frame. @@ -526,9 +526,9 @@ fn no_flipped_bit_of_the_ack_yields_another_lease() { // A flip the frame's checksums do not cover is in its link addresses, which name no // lease: taken or refused, the lease is the server's or none. let taken = wire.address(A).is_some(); - assert_eq!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease().is_some()), taken, "byte {index} bit {bit}"); + assert_eq!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease(wire.link).is_some()), taken, "byte {index} bit {bit}"); let offered = |lease: &toyos_dhcp::Lease| (lease.address, lease.prefix_len, lease.router, lease.dns.clone()) == (A, 24, Some(R), vec![DNS]); - assert!(wire.node.lease().is_none_or(offered), "byte {index} bit {bit}: {:?}", wire.node.lease()); + assert!(wire.node.lease(wire.link).is_none_or(offered), "byte {index} bit {bit}: {:?}", wire.node.lease(wire.link)); } } } diff --git a/system/netstack/node/tests/links.rs b/system/netstack/node/tests/links.rs new file mode 100644 index 00000000000..3a38864f65d --- /dev/null +++ b/system/netstack/node/tests/links.rs @@ -0,0 +1,355 @@ +//! A node over two links, the wireless one added first and the wired one second: each leases by +//! its own client, lookups ask the wired link's resolvers and follow the links up and down, and +//! the machine's name is probed for, held and lost on each link by itself. + +use std::net::Ipv4Addr; +use std::time::Duration; + +use etherparse::{ArpOperation, LinkSlice, NetSlice, PacketBuilder, SlicedPacket, TransportSlice}; +use toyos_dhcp::HostName; +use toyos_dns::Name; +use toyos_mdns::Host; +use toyos_net_node::{Event, IfIndex, Node, Rank}; +use toyos_net_shard::{Config, Secrets}; +use toyos_net_wire::ethernet::{IndividualMac, MacAddr}; +use toyos_net_wire::Instant; + +const WIRED: Rank = Rank(0); +const WIRELESS: Rank = Rank(1); +const ROUTER_MAC: [u8; 6] = [2, 0, 0, 0, 0, 0xfe]; +const OTHER_HOST_MAC: [u8; 6] = [2, 0, 0, 0, 0, 0xee]; +const GROUP: Ipv4Addr = Ipv4Addr::new(224, 0, 0, 251); +const GROUP_MAC: [u8; 6] = [0x01, 0x00, 0x5e, 0x00, 0x00, 0xfb]; +const MDNS: u16 = 5_353; + +/// One network: the node's link address on it, the address its server leases, and its router, +/// which is its DHCP server, and its resolver. +struct Net { + mac: [u8; 6], + addr: Ipv4Addr, + router: Ipv4Addr, + dns: Ipv4Addr, +} + +const WIRED_NET: Net = Net { mac: [2, 0, 0, 0, 0, 0x01], addr: Ipv4Addr::new(192, 0, 2, 10), router: Ipv4Addr::new(192, 0, 2, 1), dns: Ipv4Addr::new(192, 0, 2, 53) }; +const WIRELESS_NET: Net = + Net { mac: [2, 0, 0, 0, 0, 0x02], addr: Ipv4Addr::new(198, 51, 100, 10), router: Ipv4Addr::new(198, 51, 100, 1), dns: Ipv4Addr::new(198, 51, 100, 53) }; + +/// A frame the node sent, as the outside reads it. +#[derive(Clone, Debug, PartialEq, Eq)] +enum Out { + ArpRequest { sender: Ipv4Addr, target: Ipv4Addr }, + /// A DHCP message: its type and transaction id. + Dhcp { kind: u8, xid: u32 }, + Udp { source: Ipv4Addr, destination: Ipv4Addr, port: u16, payload: Vec }, + Other, +} + +fn read(frame: &[u8]) -> Out { + let packet = SlicedPacket::from_ethernet(frame).expect("Ethernet II"); + assert!(matches!(packet.link, Some(LinkSlice::Ethernet2(_)))); + match (&packet.net, &packet.transport) { + (Some(NetSlice::Arp(arp)), None) if arp.operation() == ArpOperation::REQUEST => { + let v4 = |b: &[u8]| Ipv4Addr::new(b[0], b[1], b[2], b[3]); + Out::ArpRequest { sender: v4(arp.sender_protocol_addr()), target: v4(arp.target_protocol_addr()) } + } + (Some(NetSlice::Ipv4(ip)), Some(TransportSlice::Udp(udp))) if udp.destination_port() == 67 => { + let message = udp.payload(); + let xid = u32::from_be_bytes(message[4..8].try_into().unwrap()); + let mut at = 240; + let kind = loop { + let (code, len) = (message[at], usize::from(message[at + 1])); + if code == 53 { + break message[at + 2]; + } + at += 2 + len; + }; + let _ = ip; + Out::Dhcp { kind, xid } + } + (Some(NetSlice::Ipv4(ip)), Some(TransportSlice::Udp(udp))) => { + let header = ip.header(); + Out::Udp { source: header.source_addr(), destination: header.destination_addr(), port: udp.destination_port(), payload: udp.payload().to_vec() } + } + _ => Out::Other, + } +} + +fn udp_frame(to: [u8; 6], from: [u8; 6], source: (Ipv4Addr, u16), destination: (Ipv4Addr, u16), payload: &[u8]) -> Vec { + let builder = PacketBuilder::ethernet2(from, to).ipv4(source.0.octets(), destination.0.octets(), 64).udp(source.1, destination.1); + let mut frame = Vec::with_capacity(builder.size(payload.len())); + builder.write(&mut frame, payload).unwrap(); + frame +} + +/// The router's ARP reply to the node on `net`, for `addr`, the router or the resolver. +fn arp_reply(net: &Net, addr: Ipv4Addr) -> Vec { + let mut body = vec![0, 1, 0x08, 0, 6, 4, 0, 2]; + body.extend_from_slice(&ROUTER_MAC); + body.extend_from_slice(&addr.octets()); + body.extend_from_slice(&net.mac); + body.extend_from_slice(&net.addr.octets()); + [&net.mac[..], &ROUTER_MAC, &[0x08, 0x06], &body].concat() +} + +/// `net`'s server's `kind` message for transaction `xid`, offering or acknowledging its address +/// for an hour, with its router and resolver. +fn dhcp(net: &Net, kind: u8, xid: u32) -> Vec { + let mut m = vec![2, 1, 6, 0]; + m.extend_from_slice(&xid.to_be_bytes()); + m.extend_from_slice(&[0; 8]); + m.extend_from_slice(&net.addr.octets()); + m.extend_from_slice(&[0; 8]); + m.extend_from_slice(&net.mac); + m.resize(236, 0); + m.extend_from_slice(&[0x63, 0x82, 0x53, 0x63]); + let options: [(u8, Vec); 6] = [ + (53, vec![kind]), + (54, net.router.octets().to_vec()), + (51, 3_600u32.to_be_bytes().to_vec()), + (1, vec![255, 255, 255, 0]), + (3, net.router.octets().to_vec()), + (6, net.dns.octets().to_vec()), + ]; + for (code, value) in options { + m.push(code); + m.push(u8::try_from(value.len()).unwrap()); + m.extend_from_slice(&value); + } + m.push(255); + udp_frame(net.mac, ROUTER_MAC, (net.router, 67), (net.addr, 68), &m) +} + +struct Two { + node: Node, + wired: IfIndex, + wireless: IfIndex, + now: Instant, + draws: u32, + /// Every frame the node sent, with the link it left by. + sent: Vec<(IfIndex, Out)>, +} + +fn counter(draws: &mut u32) -> impl FnMut() -> u32 + '_ { + move || { + *draws += 1; + *draws + } +} + +impl Two { + fn new() -> Self { + let secrets = Secrets { + ip: [1; 16], + resets: [2; 16], + tcp: toyos_net_tcp::Secrets { isn: [3; 16], timestamp: [4; 16], port_offset: [5; 16], port_index: [6; 16], port_table: [0; 16] }, + }; + let now = Instant::from_millis(3_600_000); + let mut draws = 0x4c00_0000; + let mut node = Node::new(now, Config { receive_buffer: 65_535, send_buffer: 65_535, secrets }, HostName::new("toyos")).unwrap(); + node.set_places(8); + let wireless = node.add_link(now, IndividualMac::new(MacAddr(WIRELESS_NET.mac)).unwrap(), WIRELESS, counter(&mut draws)); + let wired = node.add_link(now, IndividualMac::new(MacAddr(WIRED_NET.mac)).unwrap(), WIRED, counter(&mut draws)); + Self { node, wired, wireless, now, draws, sent: Vec::new() } + } + + fn net(&self, link: IfIndex) -> &'static Net { + if link == self.wired { &WIRED_NET } else { &WIRELESS_NET } + } + + fn deliver(&mut self, link: IfIndex, frame: &[u8]) { + let mut frames = Some(frame); + self.node.receive(self.now, link, 256, |sink| frames.take().map(&mut *sink).is_some(), counter(&mut self.draws)); + self.pump(); + } + + /// Every frame the node will send with all the credit it wants, the router answering each + /// ARP request for its own or its resolver's address, on the link it came by. + fn pump(&mut self) { + loop { + let mut out = Vec::new(); + self.node.transmit(self.now, usize::MAX, |link, frame| out.push((link, read(frame))), counter(&mut self.draws)); + if out.is_empty() { + return; + } + let replies: Vec<(IfIndex, Vec)> = out + .iter() + .filter_map(|(link, seen)| { + let net = self.net(*link); + match seen { + Out::ArpRequest { target, .. } if *target == net.router || *target == net.dns => Some((*link, arp_reply(net, *target))), + _ => None, + } + }) + .collect(); + self.sent.extend(out); + for (link, reply) in replies { + let mut frames = Some(reply.as_slice()); + self.node.receive(self.now, link, 256, |sink| frames.take().map(&mut *sink).is_some(), counter(&mut self.draws)); + } + } + } + + fn link(&mut self, link: IfIndex, up: bool) { + self.node.link(self.now, link, up, counter(&mut self.draws)); + self.pump(); + } + + /// Fires deadline after deadline until `done`, giving up past `limit`. + fn run_until(&mut self, limit: Duration, mut done: impl FnMut(&mut Two) -> bool) -> bool { + let end = self.now.after(limit); + loop { + if done(self) { + return true; + } + match self.node.next_deadline() { + Some(at) if at <= end => { + self.now = self.now.max(at); + self.node.fire(self.now, counter(&mut self.draws)); + self.pump(); + } + _ => return false, + } + } + } + + /// The last DHCP message of `kind` that left by `link`: its transaction id. + fn last_dhcp(&self, link: IfIndex, kind: u8) -> u32 { + self.sent.iter().rev().find_map(|(at, out)| match out { + Out::Dhcp { kind: k, xid } if *at == link && *k == kind => Some(*xid), + _ => None, + }) + .expect("a DHCP message of that type on the link") + } + + /// `link` up, and its server's offer and acknowledgement through to a lease held. + fn lease(&mut self, link: IfIndex) { + self.link(link, true); + let net = self.net(link); + let xid = self.last_dhcp(link, 1); + self.deliver(link, &dhcp(net, 2, xid)); + assert_eq!(self.last_dhcp(link, 3), xid, "the REQUEST"); + self.deliver(link, &dhcp(net, 5, xid)); + assert!(self.run_until(Duration::from_secs(10), |two| two.node.lease(link).is_some()), "the lease is held"); + } + + fn leased() -> Self { + let mut two = Self::new(); + let (wired, wireless) = (two.wired, two.wireless); + two.lease(wireless); + two.lease(wired); + two + } + + /// The DNS queries that left from `from`, each with its link and resolver. + fn queries(&self, from: usize) -> Vec<(IfIndex, Ipv4Addr)> { + self.sent[from..] + .iter() + .filter_map(|(link, out)| match out { + Out::Udp { destination, port: 53, .. } => Some((*link, *destination)), + _ => None, + }) + .collect() + } + + fn resolve(&mut self, text: &str) { + let name = Name::parse(text).expect("a host name"); + self.node.resolve(self.now, b"view", name, counter(&mut self.draws)).expect("a lookup started"); + self.pump(); + } +} + +#[test] +fn each_link_leases_its_own_address_by_its_own_client() { + let two = Two::leased(); + let discovers: Vec<_> = two.sent.iter().filter(|(_, out)| matches!(out, Out::Dhcp { kind: 1, .. })).collect(); + assert!(discovers.iter().any(|(link, _)| *link == two.wired) && discovers.iter().any(|(link, _)| *link == two.wireless)); + assert_ne!(two.last_dhcp(two.wired, 1), two.last_dhcp(two.wireless, 1), "two exchanges"); + assert_eq!(two.node.lease(two.wired).map(|l| l.address), Some(WIRED_NET.addr)); + assert_eq!(two.node.lease(two.wireless).map(|l| l.address), Some(WIRELESS_NET.addr)); +} + +#[test] +fn lookups_ask_the_wired_links_resolver_fail_over_and_return() { + let mut two = Two::leased(); + let (wired, wireless) = (two.wired, two.wireless); + assert_eq!(two.node.asked(), Some(wired), "wired before wireless, though it came second"); + let from = two.sent.len(); + two.resolve("one.example"); + assert_eq!(two.queries(from), [(wired, WIRED_NET.dns)]); + + two.link(wired, false); + assert_eq!(two.node.asked(), Some(wireless), "failed over"); + let from = two.sent.len(); + two.resolve("two.example"); + assert_eq!(two.queries(from), [(wireless, WIRELESS_NET.dns)]); + + two.link(wired, true); + assert!(two.run_until(Duration::from_secs(10), |two| two.node.asked() == Some(wired)), "the wired link verifies its lease and is asked again"); + let from = two.sent.len(); + two.resolve("three.example"); + assert_eq!(two.queries(from), [(wired, WIRED_NET.dns)]); +} + +#[test] +fn with_no_link_up_lookups_ask_the_lease_they_asked() { + let mut two = Two::leased(); + let wired = two.wired; + two.link(two.wireless, false); + two.link(wired, false); + assert_eq!(two.node.asked(), Some(wired)); +} + +const NAME: &[u8] = b"\x05toyos\x05local\x00"; + +/// Another host's response naming `address` for this machine's name. +fn says(address: Ipv4Addr) -> Vec { + let mut m = Vec::new(); + for word in [0u16, 0x8400, 0, 1, 0, 0] { + m.extend_from_slice(&word.to_be_bytes()); + } + m.extend_from_slice(NAME); + m.extend_from_slice(&[0, 1, 0x80, 1]); + m.extend_from_slice(&120u32.to_be_bytes()); + m.extend_from_slice(&[0, 4]); + m.extend_from_slice(&address.octets()); + m +} + +/// The mDNS messages to the group that left by `link`, each by its source. +fn to_group(two: &Two, link: IfIndex) -> Vec { + two.sent + .iter() + .filter_map(|(at, out)| match out { + Out::Udp { source, destination, port: MDNS, .. } if *at == link && *destination == GROUP => Some(*source), + _ => None, + }) + .collect() +} + +#[test] +fn the_name_is_probed_for_on_each_link_from_that_links_address_and_lost_on_one_alone() { + let mut two = Two::leased(); + let (wired, wireless) = (two.wired, two.wireless); + two.node.answer_as(two.now, Host::new("toyos").unwrap(), counter(&mut two.draws)).unwrap(); + two.pump(); + let mut events = Vec::new(); + let mut conflicted = false; + let claimed = two.run_until(Duration::from_secs(10), |two| { + if !conflicted && !to_group(two, wireless).is_empty() { + // Another host on the wireless network answers for the name under its probe. + let neighbour = Ipv4Addr::new(198, 51, 100, 99); + let frame = udp_frame(GROUP_MAC, OTHER_HOST_MAC, (neighbour, MDNS), (GROUP, MDNS), &says(neighbour)); + two.deliver(wireless, &frame); + conflicted = true; + } + events.extend(two.node.drain_events()); + events.contains(&Event::Name { link: wired, event: toyos_mdns::Event::Claimed }) + }); + assert!(claimed, "the wired link's name is claimed: {events:?}"); + assert!(events.contains(&Event::Name { link: wireless, event: toyos_mdns::Event::Lost }), "lost on the wireless link: {events:?}"); + assert!(!events.contains(&Event::Name { link: wireless, event: toyos_mdns::Event::Claimed })); + assert!(to_group(&two, wired).iter().all(|&source| source == WIRED_NET.addr)); + assert!(to_group(&two, wireless).iter().all(|&source| source == WIRELESS_NET.addr)); + assert!(!to_group(&two, wired).is_empty(), "probed for on the wired link"); +} diff --git a/system/netstack/node/tests/listeners.rs b/system/netstack/node/tests/listeners.rs index 6b9b180bdcf..672fd2d69e2 100644 --- a/system/netstack/node/tests/listeners.rs +++ b/system/netstack/node/tests/listeners.rs @@ -150,6 +150,7 @@ fn draw(draws: &mut u32) -> impl FnMut() -> u32 + '_ { struct Net { node: Node, + link: toyos_net_node::IfIndex, now: Instant, draws: u32, /// The addresses a peer is at: 192.0.2.2, and each one a SYN has come from. @@ -161,8 +162,8 @@ struct Net { impl Net { /// The node holding its lease of 192.0.2.1/24 for an hour, with `common::PLACES` places. fn new() -> Self { - let Wire { node, now, .. } = Wire::leased(&terms(3_600, Some(R))); - Self { node, now, draws: 0x7e00_0000, at: BTreeSet::from([B]), heard: Vec::new() } + let Wire { node, link, now, .. } = Wire::leased(&terms(3_600, Some(R))); + Self { node, link, now, draws: 0x7e00_0000, at: BTreeSet::from([B]), heard: Vec::new() } } /// Offers the node all the credit it wants until it sends nothing more, its ARP requests @@ -170,13 +171,13 @@ impl Net { fn pump(&mut self) { for _ in 0..100_000 { let mut frames = Vec::new(); - self.node.transmit(self.now, usize::MAX, |frame| frames.push(frame.to_vec()), draw(&mut self.draws)); + self.node.transmit(self.now, usize::MAX, |_, frame| frames.push(frame.to_vec()), draw(&mut self.draws)); if frames.is_empty() { return; } for frame in &frames { if let Some(answer) = self.hears(frame) { - self.node.receive(self.now, BUDGET, batch(&[&answer]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[&answer]), draw(&mut self.draws)); } } } @@ -212,7 +213,7 @@ impl Net { } fn deliver(&mut self, frame: &[u8]) { - self.node.receive(self.now, BUDGET, batch(&[frame]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[frame]), draw(&mut self.draws)); self.pump(); } diff --git a/system/netstack/node/tests/name.rs b/system/netstack/node/tests/name.rs index 888ac14cdbd..93ddb9a2e87 100644 --- a/system/netstack/node/tests/name.rs +++ b/system/netstack/node/tests/name.rs @@ -54,7 +54,7 @@ fn named() -> Lan { /// What the node logged of its name since the last call, in order. fn said(lan: &mut Lan) -> Vec { let name = |event| match event { - toyos_net_node::Event::Name(event) => Some(event), + toyos_net_node::Event::Name { event, .. } => Some(event), _ => None, }; lan.node.drain_events().filter_map(name).collect() @@ -256,7 +256,7 @@ fn a_held_lease_is_probed_for_three_times_and_its_name_announced_only_then() { let mut lan = named(); lan.acknowledge(&terms(3_600, Some(R))); assert!(lan.datagrams().is_empty(), "an address under probe has no name yet"); - assert!(lan.run_until(Duration::from_secs(10), |lan| lan.node.lease().is_some()), "the lease is held"); + assert!(lan.run_until(Duration::from_secs(10), |lan| lan.node.lease(lan.link).is_some()), "the lease is held"); let held = lan.now; assert_eq!(said(&mut lan), [], "the name is not this machine's yet"); let asked = to_group(MAC_B, (B, 53_000), &ours(7, CLASS_IN)); @@ -301,7 +301,7 @@ fn the_first_probe_follows_the_delay_the_callers_draw_gives() { lan.fire(quiet); lan.link(false); let back = lan.now; - lan.node.link(back, true, || 1_000); + lan.node.link(back, lan.link, true, || 1_000); lan.pump(); assert_eq!(lan.datagrams().len(), 5, "nothing before the delay is over"); until(&mut lan, 6, SECOND); @@ -325,7 +325,7 @@ fn a_held_name_is_probed_for_again_when_the_link_returns_and_announced_only_then lan.link(true); let back = lan.now; - assert!(lan.node.lease().is_some(), "the lease outlived the link"); + assert!(lan.node.lease(lan.link).is_some(), "the lease outlived the link"); until(&mut lan, from + 1, SECOND); lan.deliver(&to_group(MAC_B, (B, 53_000), &ours(7, CLASS_IN))); assert_eq!(since(&lan, from), [multicast(probe())], "probed for, and answered to nobody under the probe"); @@ -612,12 +612,12 @@ fn the_name_is_answered_only_while_its_lease_is_held() { lan.deliver(&asked); assert!(lan.datagrams().is_empty(), "under probe"); - assert!(lan.run_until(Duration::from_secs(10), |lan| lan.node.lease().is_some()), "the lease is held"); + assert!(lan.run_until(Duration::from_secs(10), |lan| lan.node.lease(lan.link).is_some()), "the lease is held"); until(&mut lan, 5, Duration::from_secs(3)); lan.deliver(&asked); assert_eq!(since(&lan, 5), [unicast(53_000, legacy_response(7))], "held, and its name claimed"); - assert!(lan.run_until(Duration::from_secs(700), |lan| lan.node.lease().is_none()), "the lease runs out"); + assert!(lan.run_until(Duration::from_secs(700), |lan| lan.node.lease(lan.link).is_none()), "the lease runs out"); let from = lan.datagrams().len(); lan.deliver(&asked); assert!(since(&lan, from).is_empty(), "gone"); diff --git a/system/netstack/node/tests/resolve.rs b/system/netstack/node/tests/resolve.rs index 2ccd9389f36..7be89c32224 100644 --- a/system/netstack/node/tests/resolve.rs +++ b/system/netstack/node/tests/resolve.rs @@ -225,7 +225,7 @@ impl Net { fn leased_for(seconds: u32, servers: &[Ipv4Addr], router: Option) -> Self { let mut lan = Lan::new(); lan.lease_on(&lease_of(seconds, servers, router)); - assert_eq!(lan.node.lease().expect("held").dns, servers, "the premise: the lease names these resolvers"); + assert_eq!(lan.node.lease(lan.link).expect("held").dns, servers, "the premise: the lease names these resolvers"); // `toyos-dns` keeps whole milliseconds and conflict detection's waits are drawn finer: the // origin is the next whole one, so a wait counted from it ends on the millisecond named. let whole = u64::try_from(lan.now.since(Instant::from_millis(0)).as_millis()).unwrap() + 1; @@ -741,7 +741,7 @@ fn a_lookup_with_no_resolver_to_ask_is_refused_as_not_connected() { let mut lan = Lan::new(); lan.lease(600); lan.node.resolve(lan.now, ONE, name("www.example"), sequence([1, 2])).expect("the premise: a held lease's resolver is asked"); - assert!(lan.run_until(Duration::from_secs(700), |lan| lan.node.lease().is_none()), "the lease runs out"); + assert!(lan.run_until(Duration::from_secs(700), |lan| lan.node.lease(lan.link).is_none()), "the lease runs out"); assert_eq!(lan.node.resolve(lan.now, ONE, name("www.example"), undrawn), Err(NotStarted::NotConnected), "after the lease"); } @@ -837,14 +837,14 @@ fn a_query_with_no_port_to_leave_from_ends_its_lookup_by_name() { fn a_burst_at_a_resolver_not_yet_resolved_is_on_the_wire_once_the_resolver_answers_arp() { let mut net = Net::leased(&[ANSWERS], Some(R)); let names: Vec = (0..MAX_LOOKUPS).map(|n| format!("n{n}.example")).collect(); - let ip = |net: &Net| net.lan.node.shard().ip().neighbour(net.lan.node.shard().iface(), ANSWERS).map(toyos_net_ip::Nud::mac); + let ip = |net: &Net| net.lan.node.shard().ip().neighbour(net.lan.link, ANSWERS).map(toyos_net_ip::Nud::mac); assert_eq!(ip(&net), None, "the premise: [ip] holds no entry for the resolver"); for asked in &names { net.resolve(asked).unwrap(); } assert!(net.queried.is_empty(), "the premise: nothing has left"); assert_eq!(net.lan.opportunity(), 1, "the request for the resolver, and no query"); - let held = match net.lan.node.shard().ip().neighbour(net.lan.node.shard().iface(), ANSWERS) { + let held = match net.lan.node.shard().ip().neighbour(net.lan.link, ANSWERS) { Some(toyos_net_ip::Nud::Incomplete(asking)) => Some(asking.pending.queued()), _ => None, }; @@ -1009,7 +1009,7 @@ fn the_query_asked_in_a_dropped_querys_place_leaves_in_the_same_opportunity() { assert_eq!(net.run(first, 10 * WAIT_MS), Some(Ok(vec![ADDRESS]))); net.until(3_000); let shard = net.lan.node.shard(); - assert!(matches!(shard.ip().neighbour(shard.iface(), SILENT), Some(toyos_net_ip::Nud::Failed)), "the premise: [ip] gave the first resolver up"); + assert!(matches!(shard.ip().neighbour(net.lan.link, SILENT), Some(toyos_net_ip::Nud::Failed)), "the premise: [ip] gave the first resolver up"); let asked = net.queried.len(); let id = net.resolve("other.example").unwrap(); @@ -1052,7 +1052,7 @@ fn lease_of(seconds: u32, servers: &[Ipv4Addr], router: Option) -> Vec #[test] fn a_lookup_outlives_a_renewal_that_names_the_same_resolvers() { let (mut net, id, renewal) = renewing_under_a_lookup(); - let expiry = |net: &Net| net.lan.node.lease().and_then(|lease| lease.timers).map(|timers| timers.expiry); + let expiry = |net: &Net| net.lan.node.lease(net.lan.link).and_then(|lease| lease.timers).map(|timers| timers.expiry); let before = expiry(&net); net.lan.deliver(&from_server(MAC, A, &message_of(ACK, renewal, &naming(ANSWERS)))); assert_ne!(expiry(&net), before, "the premise: the renewal was taken, and the lease runs from it"); @@ -1068,7 +1068,7 @@ fn a_lookup_outlives_a_renewal_that_names_the_same_resolvers() { fn a_lookup_ends_when_its_leases_resolvers_change() { let (mut net, id, renewal) = renewing_under_a_lookup(); net.lan.deliver(&from_server(MAC, A, &message_of(ACK, renewal, &naming(SILENT)))); - assert_eq!(net.lan.node.lease().expect("held").dns, [SILENT], "the premise: the renewed lease names another resolver"); + assert_eq!(net.lan.node.lease(net.lan.link).expect("held").dns, [SILENT], "the premise: the renewed lease names another resolver"); assert_eq!(net.lan.node.take_resolved(), [Resolved { id, result: Err(Ended::LeaseChanged) }]); let left = net.queried[0].udp.source_port; assert!(!net.port_held(left), "port {left} of the ended lookup"); @@ -1079,7 +1079,7 @@ fn a_lookup_ends_when_its_leases_resolvers_change() { fn a_lookup_ends_when_its_lease_is_lost() { let (mut net, id, renewal) = renewing_under_a_lookup(); net.lan.deliver(&from_server(common::BROADCAST, Ipv4Addr::BROADCAST, &message(NAK, renewal, Ipv4Addr::UNSPECIFIED, &[(54, R.octets().to_vec())]))); - assert_eq!(net.lan.node.lease(), None, "the premise: the lease is gone"); + assert_eq!(net.lan.node.lease(net.lan.link), None, "the premise: the lease is gone"); assert_eq!(net.lan.node.take_resolved(), [Resolved { id, result: Err(Ended::LeaseChanged) }]); } @@ -1166,7 +1166,7 @@ fn a_link_that_goes_down_empties_what_every_view_keeps_though_the_lease_stays() net.until(100_000); let requests = net.lan.sent[up..].iter().filter(|(_, seen)| matches!(seen, Seen::Udp(udp) if udp.port == 67)).count(); assert_eq!(requests, usize::try_from(toyos_dhcp::limits::REQUEST_TRANSMISSIONS).unwrap(), "the premise: every INIT-REBOOT went unanswered"); - assert_eq!(net.lan.node.lease().map(|lease| lease.dns.clone()), Some(vec![ANSWERS]), "the premise: the lease is in use, its resolver the same"); + assert_eq!(net.lan.node.lease(net.lan.link).map(|lease| lease.dns.clone()), Some(vec![ANSWERS]), "the premise: the lease is in use, its resolver the same"); asks_again(&mut net); } @@ -1176,14 +1176,14 @@ fn a_link_that_goes_down_empties_what_every_view_keeps_though_the_lease_stays() fn a_lease_lost_and_another_naming_the_same_resolver_empty_what_every_view_keeps() { let (mut net, renewal) = renewing(); net.lan.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message(NAK, renewal, Ipv4Addr::UNSPECIFIED, &[(54, R.octets().to_vec())]))); - assert_eq!(net.lan.node.lease(), None, "the premise: the lease is gone"); + assert_eq!(net.lan.node.lease(net.lan.link), None, "the premise: the lease is gone"); let discover = last_dhcp(&net); assert_eq!(option(&discover, 53), Some(&[DISCOVER][..])); let terms = lease_of(600, &[ANSWERS], Some(R)); net.lan.deliver(&from_server(MAC, A, &message_of(OFFER, xid(&discover), &terms))); net.lan.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, xid(&discover), &terms))); - assert!(net.lan.run_until(Duration::from_secs(10), |lan| lan.node.lease().is_some()), "the new lease is held"); - assert_eq!(net.lan.node.lease().expect("held").dns, [ANSWERS], "the premise: it names the same resolver"); + assert!(net.lan.run_until(Duration::from_secs(10), |lan| lan.node.lease(lan.link).is_some()), "the new lease is held"); + assert_eq!(net.lan.node.lease(net.lan.link).expect("held").dns, [ANSWERS], "the premise: it names the same resolver"); asks_again(&mut net); } @@ -1192,7 +1192,7 @@ fn a_lease_lost_and_another_naming_the_same_resolver_empty_what_every_view_keeps #[test] fn a_renewal_with_the_same_terms_empties_what_every_view_keeps() { let (mut net, renewal) = renewing(); - let expiry = |net: &Net| net.lan.node.lease().and_then(|lease| lease.timers).map(|timers| timers.expiry); + let expiry = |net: &Net| net.lan.node.lease(net.lan.link).and_then(|lease| lease.timers).map(|timers| timers.expiry); let before = expiry(&net); net.lan.deliver(&from_server(MAC, A, &message_of(ACK, renewal, &lease_of(600, &[ANSWERS], Some(R))))); assert_ne!(expiry(&net), before, "the premise: the renewal was taken"); diff --git a/system/netstack/node/tests/slirp.rs b/system/netstack/node/tests/slirp.rs index e142c646ac3..43e94e439bf 100644 --- a/system/netstack/node/tests/slirp.rs +++ b/system/netstack/node/tests/slirp.rs @@ -109,9 +109,9 @@ fn slirps_ack_is_the_lease_its_bytes_name() { let mut wire = requesting(); let requested = wire.now; wire.deliver(&hex(SLIRP_ACK)); - assert_eq!((wire.node.lease(), wire.address(LEASED)), (None, Some(AddrState::Tentative))); - assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease().is_some()), "the lease is held"); - let lease = wire.node.lease().unwrap(); + assert_eq!((wire.node.lease(wire.link), wire.address(LEASED)), (None, Some(AddrState::Tentative))); + assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease(wire.link).is_some()), "the lease is held"); + let lease = wire.node.lease(wire.link).unwrap(); assert_eq!((lease.address, lease.prefix_len, lease.router, lease.dns.as_slice(), lease.server), (LEASED, 24, Some(SERVER), &[RESOLVER][..], SERVER)); let expiry: Option = lease.timers.map(|timers| timers.expiry); assert_eq!(expiry, Some(requested.after(Duration::from_secs(86_400)))); @@ -133,6 +133,6 @@ fn the_lease_is_read_from_the_acks_own_mask() { // With none (RFC 768), the changed mask is the lease's. ack[UDP_CHECKSUM].fill(0); wire.deliver(&ack); - assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease().is_some()), "the lease is held"); - assert_eq!(wire.node.lease().map(|lease| lease.prefix_len), Some(23)); + assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease(wire.link).is_some()), "the lease is held"); + assert_eq!(wire.node.lease(wire.link).map(|lease| lease.prefix_len), Some(23)); } diff --git a/system/netstack/node/tests/streams.rs b/system/netstack/node/tests/streams.rs index 5a6d2c1d97b..cad6c41ade1 100644 --- a/system/netstack/node/tests/streams.rs +++ b/system/netstack/node/tests/streams.rs @@ -19,7 +19,7 @@ use std::time::Duration; use common::client::Client; use common::{arp, batch, terms, Segment, Wire, A, BUDGET, MAC, MAC_B, MAC_R, R}; use etherparse::{ArpOperation, LinkSlice, NetSlice, PacketBuilder, SlicedPacket, TcpOptionElement, TransportSlice}; -use toyos::net::stream::RESET; +use toyos::net::stream::{ADDRESS_GONE, RESET}; use toyos_net_node::{ConnectRefused, ExportRefused, Node, StreamEvent, StreamId}; use toyos_net_shard::ConnectError; use toyos_net_tcp::{Counter, Endpoint, Failure}; @@ -257,6 +257,7 @@ impl Far { struct Net { node: Node, + link: toyos_net_node::IfIndex, now: Instant, draws: u32, far: Far, @@ -276,8 +277,8 @@ fn draw(draws: &mut u32) -> impl FnMut() -> u32 + '_ { impl Net { /// The node holding its lease of 192.0.2.1/24 for an hour. fn new() -> Self { - let Wire { node, now, .. } = Wire::leased(&terms(3_600, Some(R))); - Self { node, now, draws: 0x7c00_0000, far: Far::new(), parked: Vec::new(), clients: Vec::new() } + let Wire { node, link, now, .. } = Wire::leased(&terms(3_600, Some(R))); + Self { node, link, now, draws: 0x7c00_0000, far: Far::new(), parked: Vec::new(), clients: Vec::new() } } /// Leaves the present peer with the connection it has and makes `next` the peer the test @@ -321,17 +322,17 @@ impl Net { /// the opportunity is over. Returns how many frames left. fn opportunity(&mut self) -> usize { let mut frames = Vec::new(); - self.node.transmit(self.now, usize::MAX, |frame| frames.push(frame.to_vec()), draw(&mut self.draws)); + self.node.transmit(self.now, usize::MAX, |_, frame| frames.push(frame.to_vec()), draw(&mut self.draws)); for frame in &frames { for answer in self.hears(frame) { - self.node.receive(self.now, BUDGET, batch(&[&answer]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[&answer]), draw(&mut self.draws)); } } frames.len() } fn deliver(&mut self, frame: &[u8]) { - self.node.receive(self.now, BUDGET, batch(&[frame]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[frame]), draw(&mut self.draws)); self.pump(); } @@ -532,7 +533,7 @@ fn a_batch_of_frames_moves_an_established_stream_once_with_no_opportunity_after_ assert_eq!(client.read(64), Err(Refused::WouldBlock), "the read arms its want"); let frames = [net.far.text(b"all "), net.far.text(b"at "), net.far.text(b"once")]; let rung = client.rung(); - net.node.receive(net.now, BUDGET, batch(&[&frames[0], &frames[1], &frames[2]]), draw(&mut net.draws)); + net.node.receive(net.now, net.link, BUDGET, batch(&[&frames[0], &frames[1], &frames[2]]), draw(&mut net.draws)); assert_eq!(client.rung() - rung, 1, "one ring for the batch"); assert_eq!(inbox(&client), b"all at once"); } @@ -550,13 +551,13 @@ fn a_batch_stops_at_its_budget_and_says_a_pass_is_owed() { sink(&frame); true }; - let owed = net.node.receive(net.now, BUDGET, endless, draw(&mut net.draws)); + let owed = net.node.receive(net.now, net.link, BUDGET, endless, draw(&mut net.draws)); assert_eq!((owed, handed), (true, BUDGET), "the batch is the budget, and a pass is owed"); assert_eq!(inbox(&client), b"again", "the rest of the batch was the same segment again"); let next = net.far.text(b"once"); let short: Vec<&[u8]> = vec![&next; BUDGET - 1]; - assert!(!net.node.receive(net.now, BUDGET, batch(&short), draw(&mut net.draws)), "the ring was emptied"); + assert!(!net.node.receive(net.now, net.link, BUDGET, batch(&short), draw(&mut net.draws)), "the ring was emptied"); assert_eq!(inbox(&client), b"once"); } @@ -830,6 +831,23 @@ fn a_reset_after_the_clients_shutdown_is_the_end_the_client_reads() { assert_eq!(reads(&client), (Vec::new(), Err(Refused::Failed(RESET)))); } +// A connection cannot move to another address, so one whose address the machine loses ends +// at once, named for it, rather than retransmitting until it gives up: here another host takes +// the address after one defence (RFC 5227 §2.4 (b)). Nothing is sent from the lost address. +#[test] +fn a_stream_whose_address_is_lost_ends_as_address_gone() { + let (mut net, _, client) = established(); + let conflict = arp([0xff; 6], false, MAC_B, A, A); + net.fire(net.now.after(Duration::from_secs(100))); + net.deliver(&conflict); + net.fire(net.now.after(Duration::from_secs(5))); + let heard = net.far.segments.len(); + net.deliver(&conflict); + assert_eq!(net.node.lease(net.link), None, "the premise: the address is lost"); + assert_eq!(reads(&client), (Vec::new(), Err(Refused::Failed(ADDRESS_GONE)))); + assert_eq!(net.far.segments.len(), heard, "no reset from an address the machine gave up"); +} + // RFC 9293 §3.10.7.4 and §3.10.2: a reset after the peer's FIN flushes what was queued, and a // write after it, with room in the send ring, answers the reset and copies nothing. #[test] @@ -892,7 +910,7 @@ fn a_reset_in_the_batch_of_the_text_before_it_is_read_after_the_text() { let (mut net, _, client) = established(); let text = net.far.text(b"last"); let rst = net.far.rst(); - net.node.receive(net.now, BUDGET, batch(&[&text, &rst]), draw(&mut net.draws)); + net.node.receive(net.now, net.link, BUDGET, batch(&[&text, &rst]), draw(&mut net.draws)); assert_eq!(reads(&client), (b"last".to_vec(), Err(Refused::Failed(RESET)))); } diff --git a/system/netstack/shard/src/lib.rs b/system/netstack/shard/src/lib.rs index 1fa84ae49ee..39dc48fd3bf 100644 --- a/system/netstack/shard/src/lib.rs +++ b/system/netstack/shard/src/lib.rs @@ -726,6 +726,13 @@ impl Shard { Ok(id) } + /// [`Udp::send_via`](toyos_net_udp::Udp::send_via): a datagram that leaves by `iface` alone. + pub fn send_via(&mut self, now: Instant, id: SocketId, iface: IfIndex, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), toyos_net_udp::Error> { + let queued = self.udp.send_via(&mut self.ip, id, iface, destination, port, payload); + self.settle(now); + queued + } + /// The acquisition socket's send, naming its source, 0.0.0.0 to the limited broadcast or an /// assigned address, and the interface whose address it acquires. pub fn send_from(&mut self, now: Instant, id: SocketId, origin: Origin, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), toyos_net_udp::Error> { diff --git a/system/netstack/src/serve.rs b/system/netstack/src/serve.rs index fed88b0a697..e3f2ce1ffc6 100644 --- a/system/netstack/src/serve.rs +++ b/system/netstack/src/serve.rs @@ -281,6 +281,8 @@ fn connect_failed(failure: Failure) -> u32 { Failure::Refused => ERR_CONNECTION_REFUSED, Failure::Reset => ERR_CONNECTION_RESET, Failure::TimedOut => ERR_TIMED_OUT, + // The address it was made from went under it: as no address at all. + Failure::AddressGone => ERR_NOT_CONNECTED, Failure::Unreachable(_) | Failure::Prohibited => { say!("netstack: a connect ended {failure:?}"); ERR_OTHER diff --git a/system/netstack/udp/src/lib.rs b/system/netstack/udp/src/lib.rs index 5b47f0abac9..4190ee9fa1b 100644 --- a/system/netstack/udp/src/lib.rs +++ b/system/netstack/udp/src/lib.rs @@ -518,14 +518,14 @@ impl Udp { /// Queues `payload` for `destination:port`. Accepted means queued, not sent. pub fn send_to(&mut self, ip: &mut Ip, id: SocketId, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), Error> { - self.submit(ip, id, None, destination, port, payload) + self.submit(ip, id, (None, None), destination, port, payload) } /// Queues `payload` for the connected peer. pub fn send(&mut self, ip: &mut Ip, id: SocketId, payload: &[u8]) -> Result<(), Error> { let socket = self.socket(id)?; match socket.binding { - Binding::Connected { peer, peer_port, .. } => self.submit(ip, id, None, peer, peer_port.get(), payload), + Binding::Connected { peer, peer_port, .. } => self.submit(ip, id, (None, None), peer, peer_port.get(), payload), Binding::Any | Binding::Specific(_) => { self.count(Counter::NotConnected); Err(Error::Refused(Counter::NotConnected)) @@ -536,12 +536,18 @@ impl Udp { /// The acquisition socket's send, naming its source: 0.0.0.0 to the limited broadcast, or an /// assigned address. pub fn send_from(&mut self, ip: &mut Ip, id: SocketId, origin: Origin, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), Error> { - self.submit(ip, id, Some((origin.source, origin.iface)), destination, port, payload) + self.submit(ip, id, (Some(origin.source), Some(origin.iface)), destination, port, payload) } - fn submit(&mut self, ip: &mut Ip, id: SocketId, named: Option<(Ipv4Addr, IfIndex)>, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), Error> { - let iface = named.map(|(_, iface)| iface); - let named = named.map(|(source, _)| source); + /// A datagram that leaves by `iface` alone, from the address the socket's binding and that + /// interface give it: one host's datagram to a group on each of its links. + pub fn send_via(&mut self, ip: &mut Ip, id: SocketId, iface: IfIndex, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), Error> { + self.submit(ip, id, (None, Some(iface)), destination, port, payload) + } + + /// `from` is the source a send names and the interface it names, either absent. + fn submit(&mut self, ip: &mut Ip, id: SocketId, from: (Option, Option), destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), Error> { + let (named, iface) = from; let socket = self.socket(id)?; if let Some(error) = socket.pending.take() { return Err(Error::Failed(error)); From 108f3b8947535faca555735976f40d14cc798b6a Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 31 Oct 2026 19:45:44 +0100 Subject: [PATCH 3/9] netstack runs every link a driver brings, from the start (P5) netstack holds up to eight links, each the node's interface for the boot and ranked by its hello's medium, wired before wireless, then by arrival. A hello with a new address adds a link; one with an ended link's address is opened on it again; one with an open link's address is refused Open, and one past eight Full. The node is built before any link and serves from there, so LINK_BOUND and its exit are gone: a machine with no NIC driver keeps its netstack and says, at the lease bound, that it has no address. One transmit opportunity offers the node every open link's ring room. The lease lines name their link; inspect puts each link under net.link. (its driver, medium, state, lease, netstack's own count of frames sent and the driver's counters) and the asked resolvers' link as net.dns.link. osbench sums the links' card counters and takes a connection's medium from the link holding its address. A guest test boots two virtio NICs on two QEMU user networks (Profile::HeadlessTwoNics) and has QMP set the first link down and up: traffic to TEST-NET-1 leaves by the first, by the second and none by the first while it is down, and by the first once it is back. issues/netstack-drives-one-card.md is closed: its rule, the owner's ruling that netstack runs every link wired first with failover, is P5's in the track. The NIC leaves first-match-selection-that-remains's virtio bullet. Co-Authored-By: Claude Opus 5.5 --- images/system.toml | 5 +- issues/first-match-selection-that-remains.md | 9 +- issues/netstack-drives-one-card.md | 43 -- issues/there-is-no-wifi.md | 14 +- system/netstack/src/link.rs | 158 +++++--- system/netstack/src/main.rs | 366 ++++++++++-------- system/netstack/src/serve.rs | 19 +- tests/common/iommu.rs | 12 +- tests/common/qemu.rs | 36 ++ tests/guest/osbench/src/sys/toyos.rs | 63 ++- tests/guest/rust-tests/Cargo.toml | 2 + .../guest/rust-tests/src/bin/link_failover.rs | 136 +++++++ tests/images/benchcase.toml | 2 +- tests/images/metalcase.toml | 4 +- tests/images/netcase.toml | 6 +- tests/toyos.rs | 65 ++++ 16 files changed, 629 insertions(+), 311 deletions(-) delete mode 100644 issues/netstack-drives-one-card.md create mode 100644 tests/guest/rust-tests/src/bin/link_failover.rs diff --git a/images/system.toml b/images/system.toml index 9e2ef83dc4b..623058a9a4c 100644 --- a/images/system.toml +++ b/images/system.toml @@ -132,9 +132,8 @@ drives = ["pci:04:03:*"] receives = ["sound-output"] syscap = ["rt"] -# The network stack, over the link a NIC driver brings on `link`; it claims no -# device. It serves once the first driver says hello, and leaves a machine -# none says hello on within 30 s. +# The network stack, over every link a NIC driver brings on `link`; it claims +# no device. It serves from the start, links or none. [programs.netstack] service = true serves = ["netstack", "link"] diff --git a/issues/first-match-selection-that-remains.md b/issues/first-match-selection-that-remains.md index e6e33da4b1c..baad4d064ad 100644 --- a/issues/first-match-selection-that-remains.md +++ b/issues/first-match-selection-that-remains.md @@ -12,10 +12,11 @@ does so visibly. Two do, and both are deliberate: - **NVMe.** blockd serves one controller, the first its row's claim names (`userland/blockd`); a machine with two NVMe drives loses the second. Making this an enumerate-all is a block-service change, not a PCI one. -- **The four virtio drivers.** Each takes the first device with its - (vendor, device) pair. A second NIC or a second GPU would be dropped. These - are QEMU-only devices — no virtio function appears on the T14 — so the - exposure is a test-shape one, and no profile declares two of anything virtio. +- **The virtio drivers but the NIC's.** Each takes the first device with its + (vendor, device) pair, and a second GPU would be dropped. These are + QEMU-only devices — no virtio function appears on the T14 — so the exposure + is a test-shape one. The NIC's driver is one process per function, and + netstack runs every link they bring: `Profile::HeadlessTwoNics` has two. Neither is a defect today. Both become one the moment a second such device is reachable, and the enumerate-all they would need now exists. diff --git a/issues/netstack-drives-one-card.md b/issues/netstack-drives-one-card.md deleted file mode 100644 index 7843af9cb34..00000000000 --- a/issues/netstack-drives-one-card.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-12 ---- - -# netstack drives one card - -netstack opens the first link a NIC driver brings, and no other: a hello with -another address, or with that address while its link is alive, is refused -`OneLink` (`system/netstack/src/main.rs`'s loop). -Its node is one interface with one DHCP client, made with the first link's -address (`system/netstack/node/src/lib.rs`). So a machine with a wired and a -wireless link uses only one of them, whichever driver says hello first. - -Because the node needs that address, netstack serves nothing before a -driver's hello: every client waits for it, and on a machine with no NIC -driver for 30 s (`LINK_BOUND`, `system/netstack/src/main.rs`), after which -netstack leaves and a client hears it gone. [ip] already -routes over several interfaces, the lower index winning a tie -(`system/netstack/ip/src/lib.rs`). - -The owner ruled for several interfaces over one chosen by the manifest. He -also ruled that "netstack runs every link a driver brings, wired first, with -failover" (#876, issuecomment-6120490125, ruling 3). - -That reverses part of `issues/first-match-selection-that-remains.md` -(`kind: rejected`), whose virtio bullet says a second virtio NIC "would be -dropped" and that "Neither is a defect today". Under ruling 3 a second NIC -dropped is this defect. - -Owner: P5 of `issues/there-is-no-wifi.md`. The drivers are out: each is a -process of its own over a link (`toyos-link`). - -**Exit**: -- one interface per link, each with its own DHCP client, lease and mDNS - responder, under one host name, and a node that serves before any link, - so `LINK_BOUND` goes; -- links ranked wired before wireless, then by arrival; -- a guest with two virtio NICs in which the first's link is set off over - QMP, and traffic leaves on the second; -- the NIC taken out of the virtio bullet of - `issues/first-match-selection-that-remains.md`. diff --git a/issues/there-is-no-wifi.md b/issues/there-is-no-wifi.md index ffc279aa16d..66b0720296e 100644 --- a/issues/there-is-no-wifi.md +++ b/issues/there-is-no-wifi.md @@ -216,15 +216,21 @@ stage 3 of `issues/the-lan-is-not-yet-production-grade.md`, whole. second each way full rate needs (https://github.com/ToyOSOrg/evidence/blob/4a8d02243d626df929fe05365e991873574e0dc2/s2/README.md). -**P5, netstack over several links.** Owner: the orchestrator; its defect is -`issues/netstack-drives-one-card.md`. Depends on: P4. +**P5, netstack over several links: built.** Owner: the orchestrator. +Depends on: P4. - **Exit:** - host tests of rank (wired before wireless, then arrival), failover and return, resolvers, mDNS per link, and a flow bound to a lost address; - a guest with two virtio NICs, in which the first's link is set off over QMP and traffic leaves on the second; - - `issues/netstack-drives-one-card.md` closed, and the NIC taken out of - the virtio bullet of `issues/first-match-selection-that-remains.md`. + - its defect closed, and the NIC taken out of the virtio bullet of + `issues/first-match-selection-that-remains.md`. +- **What it is.** One stack over every link a driver brings, up to eight: + each an interface of `ip`'s with a rank (a hello says Wired or Wireless), + its own lease, DHCP client and mDNS responder under one host name; lookups + ask the resolvers of the first link up holding a lease. A connection whose + address goes ends `AddressGone`; one whose link only goes down keeps it. + netstack serves from the start, so `LINK_BOUND` and `OneLink` are gone. - Its T14 row, both links leasing with the wired one the default route, is W6's: it needs a radio. diff --git a/system/netstack/src/link.rs b/system/netstack/src/link.rs index 0ab6bd16d40..89f6eb25325 100644 --- a/system/netstack/src/link.rs +++ b/system/netstack/src/link.rs @@ -16,7 +16,7 @@ use toyos_link::layout::{RX_DEPTH, SLOT_BYTES}; use toyos_link::region::Region; use toyos_link::state::{Counters, Group, Link}; use toyos_link::wire::{self, Hello, Refusal}; -use toyos_net_node::Node; +use toyos_net_node::{Credit, IfIndex, Node}; use toyos_net_wire::ethernet::{IndividualMac, MacAddr}; use toyos_net_wire::Instant; @@ -96,7 +96,10 @@ pub struct Session { region: Region, end: Stack, hello: Hello, - mac: IndividualMac, + /// The node's interface for the link. + iface: IfIndex, + /// Frames the stack handed the link: its own count, not the driver's. + sent: u64, /// The link as the node was last told it. said: Link, frame: Box<[u8; SLOT_BYTES]>, @@ -106,24 +109,19 @@ pub struct Session { } impl Session { - /// Open the link `hello` asked for over `region`, and say so to the - /// driver; `None` for a driver gone before it heard. - pub fn open(hello: Hello, mac: IndividualMac, conn: Connection, region: Region, groups: &[Group]) -> Option { + /// Open the link `hello` asked for over `region` as the node's `iface`, + /// and say so to the driver; `None` for a driver gone before it heard. + pub fn open(hello: Hello, iface: IfIndex, conn: Connection, region: Region, groups: &[Group]) -> Option { let mut end = Stack::new(region.words()); for group in groups { assert!(end.join(region.words(), *group), "netstack: more groups than a link holds"); } conn.try_send_bytes(wire::MSG_OPENED, &[]).ok()?; - Some(Self { conn, region, end, hello, mac, said: Link::Down, frame: Box::new([0; SLOT_BYTES]), asked: false, ring: true }) + Some(Self { conn, region, end, hello, iface, sent: 0, said: Link::Down, frame: Box::new([0; SLOT_BYTES]), asked: false, ring: true }) } - pub fn hello(&self) -> &Hello { - &self.hello - } - - /// The driver's address, as the node takes one. - pub fn mac(&self) -> IndividualMac { - self.mac + pub fn iface(&self) -> IfIndex { + self.iface } /// The link's end if its driver has hung up, read now. @@ -147,13 +145,14 @@ impl Session { self.hear()?; let link = self.end.link(self.region.words()).map_err(Ended::Fault)?; if link.is_up() != self.said.is_up() { - node.link(now, link.is_up(), draw); + node.link(now, self.iface, link.is_up(), draw); } self.said = link; let (words, bytes, end, frame) = (self.region.words(), self.region.bytes(), &mut self.end, &mut self.frame); let mut fault = None; let owed = node.receive( now, + self.iface, RX_DEPTH as usize, |sink| match end.received(words) { Ok(Some(span)) => { @@ -177,36 +176,26 @@ impl Session { } } - /// Offer the node the transmit ring's room until it has no frame left or - /// the ring no room, and ring the driver for what was sent. - pub fn transmit(&mut self, node: &mut Node, now: Instant, draw: fn() -> u32) -> Result<(), Ended> { + /// The transmit ring's room, armed to be rung for when there is none. + fn room(&mut self) -> Result { let words = self.region.words(); - loop { - let room = match self.end.room(words).map_err(Ended::Fault)? { - 0 => self.end.want_room(words).map_err(Ended::Fault)?, - room => room, - } as usize; - let (bytes, end) = (self.region.bytes(), &mut self.end); - let mut fault = None; - let sent = node.transmit( - now, - room, - |frame| { - bytes.write(end.slot().offset, frame); - if let Err(why) = end.send(words, frame.len()) { - fault.get_or_insert(why); - } - }, - draw, - ); - if let Some(why) = fault { - return Err(Ended::Fault(why)); - } - self.ring |= self.end.publish(words); - if room == 0 || sent < room { - return Ok(()); - } - } + Ok(match self.end.room(words)? { + 0 => self.end.want_room(words)?, + room => room, + } as usize) + } + + /// One frame into the transmit ring's next slot. + fn send(&mut self, frame: &[u8]) -> Result<(), Fault> { + self.region.bytes().write(self.end.slot().offset, frame); + self.end.send(self.region.words(), frame.len())?; + self.sent = self.sent.saturating_add(1); + Ok(()) + } + + /// What was sent reaches the driver, which is rung for it. + fn publish(&mut self) { + self.ring |= self.end.publish(self.region.words()); } /// Ask the driver for its counters, which [`Self::counters`] reads once @@ -261,21 +250,86 @@ impl Session { /// **An unreported link is `unreported`, not `up`**: the device tells the /// stack nothing about one, and the stack serving as though it were up is /// its assumption rather than something it measured. - pub fn inspect(&self, counters: Option<&Counters>, snap: &mut Snapshot) { + pub fn inspect(&self, at: &str, counters: Option<&Counters>, snap: &mut Snapshot) { let m = self.hello.mac.bytes(); - snap.put("mac", format!("{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", m[0], m[1], m[2], m[3], m[4], m[5])); - snap.put("driver", self.hello.label.as_str()); + snap.put(&format!("{at}.mac"), format!("{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", m[0], m[1], m[2], m[3], m[4], m[5])); + snap.put(&format!("{at}.driver"), self.hello.label.as_str()); + snap.put( + &format!("{at}.medium"), + match self.hello.medium { + wire::Medium::Wired => "wired", + wire::Medium::Wireless => "wireless", + }, + ); + snap.put(&format!("{at}.sent"), self.sent); match self.said { - Link::Down => snap.put("link.state", "down"), - Link::Unreported => snap.put("link.state", "unreported"), + Link::Down => snap.put(&format!("{at}.state"), "down"), + Link::Unreported => snap.put(&format!("{at}.state"), "unreported"), Link::Up { mbps, full_duplex } => { - snap.put("link.state", "up"); - snap.put("link.speed_mbps", mbps); - snap.put("link.duplex", if full_duplex { "full" } else { "half" }); + snap.put(&format!("{at}.state"), "up"); + snap.put(&format!("{at}.speed_mbps"), mbps); + snap.put(&format!("{at}.duplex"), if full_duplex { "full" } else { "half" }); } } for (counter, value) in counters.into_iter().flat_map(Counters::iter) { - snap.put(counter.name(), value); + snap.put(&format!("{at}.{}", counter.name()), value); + } + } +} + +/// Each open link's transmit room in one opportunity, a frame of it spent as +/// the node hands the link one. +struct Rooms(Vec<(IfIndex, usize)>); + +impl Credit for Rooms { + fn room(&self, iface: IfIndex) -> bool { + self.0.iter().any(|&(at, room)| at == iface && room > 0) + } + fn spend(&mut self, iface: IfIndex) { + if let Some((_, room)) = self.0.iter_mut().find(|(at, _)| *at == iface) { + *room = room.saturating_sub(1); + } + } +} + +/// Offer the node every open link's transmit room until it has no frame left +/// or no link room, and ring each driver for what was sent. Returns each link +/// whose driver wrote what no driver writes, which is to end. +pub fn transmit(sessions: &mut [&mut Session], node: &mut Node, now: Instant, draw: fn() -> u32) -> Vec<(IfIndex, Fault)> { + let mut faults: Vec<(IfIndex, Fault)> = Vec::new(); + loop { + let mut rooms = Rooms(Vec::new()); + for session in sessions.iter_mut() { + if faults.iter().any(|(at, _)| *at == session.iface) { + continue; + } + match session.room() { + Ok(room) => rooms.0.push((session.iface, room)), + Err(why) => faults.push((session.iface, why)), + } + } + let offered: usize = rooms.0.iter().map(|(_, room)| room).sum(); + if offered == 0 { + return faults; + } + let sent = node.transmit( + now, + &mut rooms, + |iface, frame| { + let Some(session) = sessions.iter_mut().find(|s| s.iface == iface) else { unreachable!("the node sent by a link netstack does not hold") }; + if let Err(why) = session.send(frame) + && !faults.iter().any(|(at, _)| *at == iface) + { + faults.push((iface, why)); + } + }, + draw, + ); + for session in sessions.iter_mut() { + session.publish(); + } + if sent < offered { + return faults; } } } diff --git a/system/netstack/src/main.rs b/system/netstack/src/main.rs index a4efe2052d4..b42f6a36e4e 100644 --- a/system/netstack/src/main.rs +++ b/system/netstack/src/main.rs @@ -1,5 +1,5 @@ -//! The network server: ToyOS's own stack (`toyos-net-node`) over the link a -//! NIC driver brings (`toyos-link`), served to programs over `toyos::net`'s +//! The network server: ToyOS's own stack (`toyos-net-node`) over every link +//! a NIC driver brings (`toyos-link`), served to programs over `toyos::net`'s //! protocol. //! //! **This program holds no protocol and drives no device.** Every byte off the @@ -10,18 +10,17 @@ //! holders are rung on (`bells`), the clients' connections (`client`) and //! their requests (`serve`). //! -//! **One link, until netstack runs several** -//! (`issues/netstack-drives-one-card.md`): the node is made with the first -//! driver's address, so nothing is served before that driver's hello, and a -//! machine none says hello on within [`LINK_BOUND`] has no network: netstack -//! leaves, as a client then hears. A link that ends leaves every connection -//! to the node, and only a hello with that address is opened again. +//! **Every link a driver brings**, up to [`MAX_LINKS`]: each is an interface +//! of the node's, ranked wired before wireless and then by arrival, with its +//! own lease. The node serves from the start, before any link. A link that +//! ends keeps its interface, down, and a hello with its address is opened +//! on it again; a hello with the address of a link that is open is refused. //! -//! **One pass**: the link's state and its received frames, as one batch, go to the node, then -//! every deadline that is due, then the node is offered the link's transmit -//! room until it has no frame left or the link no room; what the node has to +//! **One pass**: each link's state and its received frames, as one batch, go to the node, then +//! every deadline that is due, then the node is offered every link's transmit +//! room until it has no frame left or no link room; what the node has to //! say is written to the log and to the clients that waited for it; and the -//! loop sleeps on the kernel until the link, a client, a watched handle or the +//! loop sleeps on the kernel until a link, a client, a watched handle or the //! node's next deadline wakes it. A request is carried out in the pass that //! read it and its frames leave in the next, which follows at once. //! @@ -44,9 +43,9 @@ use toyos_abi::syscall::PIPE_CAPACITY; use toyos_dhcp::{HostName, Lease}; use toyos_inspect::Snapshot; use toyos_link::state::{Counters, Group}; -use toyos_link::wire::{Hello, Refusal}; +use toyos_link::wire::{Hello, Medium, Refusal}; use toyos_net_ip::Nud; -use toyos_net_node::Node; +use toyos_net_node::{IfIndex, Node, Rank}; use toyos_net_shard::{Config, Secrets}; use toyos_net_wire::ethernet::IndividualMac; use toyos_net_wire::Instant; @@ -61,13 +60,11 @@ mod watches; use client::{Client, ClientRx, HANDSHAKE_TIMEOUT, MAX_PENDING_CONNS, PendingConn, Request}; use link::{Ended, Heard, Pending, Session}; -/// How long netstack waits for its first link before it leaves as a machine -/// with no network. Policy: a driver says hello once its device is up, which -/// is seconds of this; a machine with no NIC driver waits it out, and its -/// clients with it. -const LINK_BOUND: Duration = Duration::from_secs(30); +/// The most links netstack drives. Each is an interface of the node's for +/// the boot, kept for its driver's return; a machine has a handful of NICs. +const MAX_LINKS: u32 = 8; -/// How long an `inspect` waits for the driver's counters. The driver answers +/// How long an `inspect` waits for the drivers' counters. A driver answers /// in the pass the ask's doorbell starts; one that does not is answered for /// without them. const INSPECT_BOUND: Duration = Duration::from_secs(1); @@ -80,7 +77,7 @@ const MAX_PENDING_LINKS: u32 = 2; /// one machine. pub const HOSTNAME: &str = "toyos-t14"; -/// How long this machine waits for its first lease before saying it has none. +/// How long this machine waits for its first lease, on any link, before saying it has none. /// It bounds the report, never the client: the node asks for the life of the /// boot and a lease that lands later is applied like any other. const LEASE_BOUND: Duration = Duration::from_millis(toyos_tco::LEASE_BOUND_MS); @@ -129,9 +126,9 @@ const PLACE_BYTES: u64 = { /// this in favour of a kernel memory limit, not in favour of a bigger number. const PLACE_BUDGET_SHARE: u64 = 8; -/// Watches that are no place's: the service's two acceptors, the link, and +/// Watches that are no place's: the service's two acceptors, the links, and /// the drivers' connections that have not said hello. -const FIXED_WATCHES: u32 = 3 + MAX_PENDING_LINKS; +const FIXED_WATCHES: u32 = 2 + MAX_LINKS + MAX_PENDING_LINKS; /// The most places one poller can watch: each place is one watched handle, a /// holder's connection of a stream, a listener or a datagram socket, asked @@ -197,30 +194,36 @@ impl Said { } } -/// What the boot's log has said of the lease, and still owes. +/// What the boot's log has said of each link's lease, and still owes. struct Leases { began: Wall, - said: Option, - /// Whether this boot has settled the question once: a lease landed, or - /// the bound passed with none. + said: Vec<(IfIndex, Said)>, + /// Whether this boot has settled the question once: a lease landed on a + /// link, or the bound passed with none. settled: bool, } impl Leases { - /// Says what changed of the lease the node holds, and answers whether - /// this machine's address question has just been settled. - fn pass(&mut self, held: Option<&Lease>) -> bool { - let held = held.map(Said::of); - if held != self.said { + /// Says what changed of each link's lease, `label` naming the link, and + /// answers whether this machine's address question has just been + /// settled. + fn pass(&mut self, node: &Node, label: impl Fn(IfIndex) -> String) -> bool { + for link in node.links() { + let held = node.lease(link).map(Said::of); + let at = self.said.iter().position(|(on, _)| *on == link); + if held.as_ref() == at.map(|at| &self.said[at].1) { + continue; + } match &held { // One record carrying every field the lease decided: a boot // read off a stick or a stream has this line and nothing else // to say what this machine's network was. Some(lease) => say!( - "netstack: DHCP: lease {}/{} from {}, gateway {}, dns [{}], {} ms after \ + "netstack: DHCP: lease {}/{} on {}, from {}, gateway {}, dns [{}], {} ms after \ netstack came up", lease.address, lease.prefix_len, + label(link), lease.server, match lease.router { Some(router) => router.to_string(), @@ -229,20 +232,25 @@ impl Leases { lease.dns(), self.began.elapsed().as_millis(), ), - None => say!("netstack: DHCP: the lease is gone; this machine has no address"), + None => say!("netstack: DHCP: the lease on {} is gone; that link has no address", label(link)), + } + match (held, at) { + (Some(held), Some(at)) => self.said[at].1 = held, + (Some(held), None) => self.said.push((link, held)), + (None, Some(at)) => drop(self.said.remove(at)), + (None, None) => {} } - self.said = held; } if self.settled { return false; } - if self.said.is_none() { + if self.said.is_empty() { if self.began.elapsed() < LEASE_BOUND { return false; } say!( - "netstack: DHCP: no lease as {} in {} s; this machine has no address and every \ - connect through it is refused", + "netstack: DHCP: no lease as {} on any link in {} s; this machine has no address \ + and every connect through it is refused", HOSTNAME, self.began.elapsed().as_secs(), ); @@ -256,23 +264,22 @@ impl Leases { (!self.settled).then(|| LEASE_BOUND.saturating_sub(self.began.elapsed())) } - /// The lease as `inspect` reads it, and the router's entry in the - /// stack's neighbour table where the lease names one. - fn inspect(&self, node: &Node, snap: &mut Snapshot) { - let Some(held) = &self.said else { - snap.put("lease.held", false); + /// `link`'s lease as `inspect` reads it, under `at`, and the router's + /// entry in the stack's neighbour table where the lease names one. + fn inspect(&self, node: &Node, link: IfIndex, at: &str, snap: &mut Snapshot) { + let Some((_, held)) = self.said.iter().find(|(on, _)| *on == link) else { + snap.put(&format!("{at}.lease.held"), false); return; }; - snap.put("lease.held", true); - snap.put("lease.address", format!("{}/{}", held.address, held.prefix_len)); - snap.put("lease.server", held.server.to_string()); - snap.put("lease.dns", held.dns()); + snap.put(&format!("{at}.lease.held"), true); + snap.put(&format!("{at}.lease.address"), format!("{}/{}", held.address, held.prefix_len)); + snap.put(&format!("{at}.lease.server"), held.server.to_string()); + snap.put(&format!("{at}.lease.dns"), held.dns()); let Some(router) = held.router else { return }; - snap.put("lease.router", router.to_string()); - let shard = node.shard(); + snap.put(&format!("{at}.lease.router"), router.to_string()); snap.put( - "neighbour.router", - match shard.ip().neighbour(shard.iface(), router) { + &format!("{at}.neighbour.router"), + match node.shard().ip().neighbour(link, router) { None => "none", Some(Nud::Incomplete(_)) => "incomplete", Some(Nud::Reachable(_)) => "reachable", @@ -286,24 +293,59 @@ impl Leases { } } +/// A link netstack has opened this boot: its interface, kept for its driver's +/// return, and its session while the link is open. +struct Held { + iface: IfIndex, + mac: IndividualMac, + label: String, + session: Option, + /// Its last link ended for what its driver wrote: its next hello is + /// refused for it. + faulted: bool, +} + +/// Where a link stands among the others: wired before wireless. +fn rank(medium: Medium) -> Rank { + match medium { + Medium::Wired => Rank(0), + Medium::Wireless => Rank(1), + } +} + +/// `link.`, the `n`th link opened this boot, in `inspect`'s paths. +fn path(n: usize) -> String { + format!("link.{n}") +} + /// Answer `inspect` with what this pass knows, in one non-blocking write, and /// let the connection close as every other answer does. -fn answer_inspect(request: &Request, link: Option<(&Session, Option<&Counters>)>, leases: &Leases, sockets: &serve::Sockets, node: &Node) { +fn answer_inspect(request: &Request, links: &[Held], counters: &[(IfIndex, Counters)], leases: &Leases, sockets: &serve::Sockets, node: &Node) { let mut snap = Snapshot::new(toyos_inspect::NET); - match link { - Some((link, counters)) => link.inspect(counters, &mut snap), - None => snap.put("link.state", "none"), + snap.put("links", links.len() as u32); + for (n, held) in links.iter().enumerate() { + let at = path(n); + match &held.session { + Some(session) => session.inspect(&at, counters.iter().find(|(on, _)| *on == held.iface).map(|(_, c)| c), &mut snap), + None => { + snap.put(&format!("{at}.driver"), held.label.as_str()); + snap.put(&format!("{at}.state"), "ended"); + } + } + leases.inspect(node, held.iface, &at, &mut snap); + if node.asked() == Some(held.iface) { + snap.put("dns.link", n as u32); + } } - leases.inspect(node, &mut snap); sockets.inspect(node, &mut snap); let encoded = snap.encode().unwrap_or_else(|why| panic!("netstack: its snapshot: {why}")); request.client.snapshot(&encoded); } -/// Open the link `asked` asks for, and say so. -fn open(asked: Asked, groups: &[Group]) -> Option { +/// Open the link `asked` asks for on the node's `iface`, and say so. +fn open(asked: Asked, iface: IfIndex, groups: &[Group]) -> Option { let label = asked.hello.label.clone(); - let opened = Session::open(asked.hello, asked.mac, asked.conn, asked.region, groups); + let opened = Session::open(asked.hello, iface, asked.conn, asked.region, groups); match &opened { Some(_) => say!("netstack: {label}'s link is open"), None => say!("netstack: {label} went before its link was opened"), @@ -339,18 +381,56 @@ fn hear_links(pending: &mut Vec) -> Vec { asked } -/// End `link` for `why`; the node hears a change, not a state. -fn end_link(link: &mut Option, why: Ended, node: &mut Node, now: Instant, faulted: &mut bool) { - let Some(session) = link.take() else { return }; +/// End `held`'s link for `why`; the node hears a change, not a state. +fn end_link(held: &mut Held, why: Ended, node: &mut Node, now: Instant) { + let Some(session) = held.session.take() else { return }; match why { - Ended::HungUp => say!("netstack: {}'s link ended: it hung up", session.hello().label), + Ended::HungUp => say!("netstack: {}'s link ended: it hung up", held.label), Ended::Fault(fault) => { - say!("netstack: {}'s link ended: it wrote {fault}", session.hello().label); - *faulted = true; + say!("netstack: {}'s link ended: it wrote {fault}", held.label); + held.faulted = true; } } if session.is_up() { - node.link(now, false, draw); + node.link(now, held.iface, false, draw); + } +} + +/// Open, refuse or take back the link each hello asks for. +fn hellos(asked: Vec, links: &mut Vec, node: &mut Node, now: Instant, groups: &[Group]) { + for asked in asked { + let at = links.iter().position(|held| held.mac == asked.mac); + let Some(at) = at else { + if links.len() >= MAX_LINKS as usize { + refuse(&asked, Refusal::Full, &format!("netstack drives at most {MAX_LINKS} links")); + continue; + } + let iface = node.add_link(now, asked.mac, rank(asked.hello.medium), draw); + let label = asked.hello.label.clone(); + let mac = asked.mac; + let session = open(asked, iface, groups); + links.push(Held { iface, mac, label, session, faulted: false }); + continue; + }; + let held = &mut links[at]; + if std::mem::take(&mut held.faulted) { + refuse(&asked, Refusal::Faulted, "its last link ended for what it wrote"); + continue; + } + // The kernel closes a dead driver's handles before its end is + // published, so a restarted driver's hello finds the old link already + // hung up. + if let Some(session) = &held.session { + match session.alive() { + Ok(()) => { + refuse(&asked, Refusal::Open, "a link of its address is open"); + continue; + } + Err(why) => end_link(held, why, node, now), + } + } + held.label = asked.hello.label.clone(); + held.session = open(asked, held.iface, groups); } } @@ -363,39 +443,11 @@ fn accept_link(links: &Acceptor, pending: &mut Vec) { } } -/// The first link a driver brings, within [`LINK_BOUND`]. -fn first_link(links: &Acceptor, groups: &[Group]) -> Option { - let began = Wall::now(); - let mut pending: Vec = Vec::new(); - let mut poller = watches::Watches::new(1 + MAX_PENDING_LINKS); - loop { - let mut opened = None; - for asked in hear_links(&mut pending) { - match opened { - Some(_) => refuse(&asked, Refusal::OneLink, "netstack drives one link (issues/netstack-drives-one-card.md)"), - None => opened = open(asked, groups), - } - } - if opened.is_some() { - return opened; - } - let left = LINK_BOUND.checked_sub(began.elapsed())?; - poller.watch(links, READABLE, TOKEN_LINKS); - for (i, p) in pending.iter().enumerate() { - p.watch(&mut poller, TOKEN_LINK_PENDING_BASE + i as u64); - } - let mut accept = false; - poller.wait_answers(1, u64::try_from(left.as_nanos()).unwrap_or(u64::MAX), |token, _| accept |= token == TOKEN_LINKS); - if accept { - accept_link(links, &mut pending); - } - } -} - const TOKEN_ACCEPTOR: u64 = 0; const TOKEN_LINKS: u64 = 1; -const TOKEN_LINK: u64 = 2; -const TOKEN_LINK_PENDING_BASE: u64 = 3; +const TOKEN_LINK_PENDING_BASE: u64 = 2; +/// The `n`th link's token. +const TOKEN_LINK_BASE: u64 = TOKEN_LINK_PENDING_BASE + MAX_PENDING_LINKS as u64; const _: () = assert!( toyos_inspect::MAX_SNAPSHOT_BYTES == ipc::MAX_FRAME_LEN as usize, @@ -410,7 +462,7 @@ struct Asked { region: toyos_link::region::Region, } -/// An `inspect` waiting for the driver's counters. +/// An `inspect` waiting for the drivers' counters. struct Inspecting { request: Request, since: Wall, @@ -426,23 +478,16 @@ fn main() { let links = endow::acceptor(toyos_link::PORT) .expect("the manifest declares this program serves `link`"); let groups = [Group::new(toyos_mdns::GROUP_MAC).unwrap_or_else(|| panic!("netstack: mDNS's address is no group's"))]; - let Some(first) = first_link(&links, &groups) else { - say!("netstack: no NIC driver brought a link in {} s; this machine has no network, exiting", LINK_BOUND.as_secs()); - return; - }; - let mac = first.mac(); - let mut link = Some(first); let began = Wall::now(); let clock = || Instant::from_nanos(u64::try_from(began.elapsed().as_nanos()).unwrap_or(u64::MAX)); let config = Config { - mac, receive_buffer: TCP_BUFFER, send_buffer: toyos::net::stream::SEND_BYTES, secrets: secrets(), }; let host = HostName::new(HOSTNAME).unwrap_or_else(|| panic!("netstack: {HOSTNAME:?} is no host name")); - let mut node = Node::new(clock(), config, Some(host), draw) + let mut node = Node::new(clock(), config, Some(host)) .unwrap_or_else(|why| panic!("netstack: the stack refused its configuration: {why:?}")); let name = toyos_mdns::Host::new(HOSTNAME).unwrap_or_else(|_| panic!("netstack: {HOSTNAME:?} is no host name")); @@ -453,7 +498,8 @@ fn main() { let places = places_for(total_mem); node.set_places(places); let mut sockets = serve::Sockets::new(places); - let mut leases = Leases { began, said: None, settled: false }; + let mut leases = Leases { began, said: Vec::new(), settled: false }; + let mut held: Vec = Vec::new(); let mut poller = watches::Watches::new( FIXED_WATCHES + MAX_PLACES + MAX_PENDING_CONNS + serve::LOOKUP_WATCHES, @@ -466,62 +512,71 @@ fn main() { let mut pending: Vec = Vec::new(); let mut pending_links: Vec = Vec::new(); let mut inspecting: Vec = Vec::new(); - // The held link ended for what its driver wrote, and its next hello is - // refused for it. - let mut faulted = false; + // What the drivers have answered of the counters `inspecting` waits for. + let mut gathered: Vec<(IfIndex, Counters)> = Vec::new(); // Accepts the kernel refused since the last it did not. let mut accept_refused: u64 = 0; loop { - // The node begins with its link down, and hears each change of it here. + // The node begins with each link down, and hears each change of it here. let now = clock(); let mut owed = false; - let mut ended = None; - if let Some(session) = &mut link { + let mut ended: Vec<(IfIndex, Ended)> = Vec::new(); + for session in held.iter_mut().filter_map(|h| h.session.as_mut()) { match session.receive(&mut node, now, draw) { - Ok(more) => owed = more, - Err(why) => ended = Some(why), + Ok(more) => owed |= more, + Err(why) => ended.push((session.iface(), why)), } } let now = clock(); if node.next_deadline().is_some_and(|at| at <= now) { node.fire(now, draw); } - if ended.is_none() - && let Some(session) = &mut link - && let Err(why) = session.transmit(&mut node, now, draw) - { - ended = Some(why); - } + let mut open: Vec<&mut Session> = + held.iter_mut().filter_map(|h| h.session.as_mut()).filter(|s| !ended.iter().any(|(at, _)| *at == s.iface())).collect(); + ended.extend(link::transmit(&mut open, &mut node, now, draw).into_iter().map(|(at, fault)| (at, Ended::Fault(fault)))); - // The driver's counters, once it answered them or its time to is up. + // The drivers' counters, once each open link's driver answered them or + // the time to is up. if !inspecting.is_empty() { - // A link opened since the ask is asked in its turn. - if let Some(session) = &mut link { + let mut owing = false; + for session in held.iter_mut().filter_map(|h| h.session.as_mut()) { + if gathered.iter().any(|(at, _)| *at == session.iface()) { + continue; + } + // A link opened since the ask is asked in its turn. session.ask_counters(); + match session.counters() { + Some(c) => gathered.push((session.iface(), c)), + None => owing = true, + } } - let counters = link.as_mut().and_then(Session::counters); let late = inspecting.first().is_some_and(|i| i.since.elapsed() >= INSPECT_BOUND); - if counters.is_some() || late || link.is_none() { + if !owing || late { for waiting in inspecting.drain(..) { - answer_inspect(&waiting.request, link.as_ref().map(|l| (l, counters.as_ref())), &leases, &sockets, &node); + answer_inspect(&waiting.request, &held, &gathered, &leases, &sockets, &node); } + gathered.clear(); } } - if ended.is_none() - && let Some(session) = &mut link - && let Err(why) = session.ring() - { - ended = Some(why); + for session in held.iter_mut().filter_map(|h| h.session.as_mut()) { + if !ended.iter().any(|(at, _)| *at == session.iface()) + && let Err(why) = session.ring() + { + ended.push((session.iface(), why)); + } } - if let Some(why) = ended { - end_link(&mut link, why, &mut node, clock(), &mut faulted); + for (iface, why) in ended { + if let Some(link) = held.iter_mut().find(|h| h.iface == iface) { + end_link(link, why, &mut node, clock()); + } } - sockets.settle(&mut node); + let label = |iface: IfIndex| held.iter().find(|h| h.iface == iface).map_or_else(String::new, |h| h.label.clone()); + sockets.settle(&mut node, label); // Before anything is served: the lease is what gives this machine an // address, a route and its resolvers. - if leases.pass(node.lease()) { + if leases.pass(&node, label) { say!( "netstack: ready, at most {places} places ({} MiB each of {} MiB total)", PLACE_BYTES / (1024 * 1024), @@ -531,8 +586,8 @@ fn main() { poller.watch(&acceptor, READABLE, TOKEN_ACCEPTOR); poller.watch(&links, READABLE, TOKEN_LINKS); - if let Some(session) = &link { - session.watch(&mut poller, TOKEN_LINK); + for (n, session) in held.iter().enumerate().filter_map(|(n, h)| h.session.as_ref().map(|s| (n, s))) { + session.watch(&mut poller, TOKEN_LINK_BASE + n as u64); } for (i, p) in pending_links.iter().enumerate() { p.watch(&mut poller, TOKEN_LINK_PENDING_BASE + i as u64); @@ -570,33 +625,12 @@ fn main() { }); sockets.bridge(&mut node, clock()); - // A driver's connection: a hello with the address the node was made - // with is opened, once a link it replaces has ended. The kernel closes - // a dead driver's handles before its end is published, so a - // restarted driver's hello finds the old link already hung up. + // A driver's connection: each hello is opened on a link of its own, or on + // its address's once that link has ended. if ready.contains(&TOKEN_LINKS) { accept_link(&links, &mut pending_links); } - for asked in hear_links(&mut pending_links) { - if asked.mac != mac { - refuse(&asked, Refusal::OneLink, "its address is not the one netstack drives (issues/netstack-drives-one-card.md)"); - continue; - } - if std::mem::take(&mut faulted) { - refuse(&asked, Refusal::Faulted, "its last link ended for what it wrote"); - continue; - } - if let Some(session) = &link { - match session.alive() { - Ok(()) => { - refuse(&asked, Refusal::OneLink, "a link of its address is open"); - continue; - } - Err(why) => end_link(&mut link, why, &mut node, clock(), &mut faulted), - } - } - link = open(asked, &groups); - } + hellos(hear_links(&mut pending_links), &mut held, &mut node, clock(), &groups); // On a pass that found nothing ready too: otherwise a silent client // is only ever timed out by some other client's traffic. let now_wall = Wall::now(); @@ -697,11 +731,13 @@ fn main() { // not this protocol. if request.payload_len != 0 { request.client.error(toyos::net::ERR_INVALID_INPUT); - } else if let Some(session) = link.as_mut().filter(|_| inspecting.len() < MAX_PENDING_CONNS as usize) { - session.ask_counters(); + } else if held.iter().any(|h| h.session.is_some()) && inspecting.len() < MAX_PENDING_CONNS as usize { + for session in held.iter_mut().filter_map(|h| h.session.as_mut()) { + session.ask_counters(); + } inspecting.push(Inspecting { request, since: Wall::now() }); } else { - answer_inspect(&request, link.as_ref().map(|l| (l, None)), &leases, &sockets, &node); + answer_inspect(&request, &held, &[], &leases, &sockets, &node); } continue; } diff --git a/system/netstack/src/serve.rs b/system/netstack/src/serve.rs index e3f2ce1ffc6..d237a5119a6 100644 --- a/system/netstack/src/serve.rs +++ b/system/netstack/src/serve.rs @@ -515,8 +515,8 @@ impl Sockets { /// Everything the node has to say since the last call, to the log and to /// the clients that waited for it, and the table entries of what the node - /// let go. - pub fn settle(&mut self, node: &mut Node) { + /// let go. `label` names a link in a line. + pub fn settle(&mut self, node: &mut Node, label: impl Fn(toyos_net_node::IfIndex) -> String) { let events: Vec = node.drain_stream_events().collect(); for event in events { let (id, answer) = match event { @@ -551,13 +551,16 @@ impl Sockets { match event { Event::Stack { refusal, suppressed: 0 } => say!("netstack: refused {refusal:?}"), Event::Stack { refusal, suppressed } => say!("netstack: refused {refusal:?}, and {suppressed} more by its rule"), - Event::Dhcp { refusal, suppressed: 0 } => say!("netstack: DHCP: refused {refusal:?}"), - Event::Dhcp { refusal, suppressed } => say!("netstack: DHCP: refused {refusal:?}, and {suppressed} more by its rule"), - Event::Name(toyos_mdns::Event::Claimed) => { - say!("netstack: mDNS: no host answered for {HOSTNAME}.local; this machine answers as it") + Event::Dhcp { link, refusal, suppressed: 0 } => say!("netstack: DHCP on {}: refused {refusal:?}", label(link)), + Event::Dhcp { link, refusal, suppressed } => { + say!("netstack: DHCP on {}: refused {refusal:?}, and {suppressed} more by its rule", label(link)) } - Event::Name(toyos_mdns::Event::Lost) => say!( - "netstack: mDNS: another host answered for {HOSTNAME}.local; this machine answers to no name and asks for {HOSTNAME}.local again every {} s", + Event::Name { link, event: toyos_mdns::Event::Claimed } => { + say!("netstack: mDNS on {}: no host answered for {HOSTNAME}.local; this machine answers as it there", label(link)) + } + Event::Name { link, event: toyos_mdns::Event::Lost } => say!( + "netstack: mDNS on {}: another host answered for {HOSTNAME}.local; this machine answers to no name there and asks for {HOSTNAME}.local again every {} s", + label(link), toyos_mdns::RETRY_MS / 1000 ), } diff --git a/tests/common/iommu.rs b/tests/common/iommu.rs index 0546c986e2f..db02b5df36f 100644 --- a/tests/common/iommu.rs +++ b/tests/common/iommu.rs @@ -518,22 +518,22 @@ fn untranslated_nic() -> String { /// A virtio NIC that bypasses the unit ([`Profile::HeadlessNicUntranslated`]) /// is never handed to its driver: the supervisor reads its offer through the -/// claim, refuses it by name, starts no NIC driver, and netstack leaves once -/// its wait for a link is out. Only a QEMU machine can make one: every bench -/// function is behind its unit. +/// claim, refuses it by name and starts no NIC driver, and netstack serves +/// with no link, saying so once its bound on a first lease is out. Only a +/// QEMU machine can make one: every bench function is behind its unit. pub fn a_virtio_function_without_access_platform_is_refused() -> Result<(), String> { let options = BootOptions { profile: Profile::HeadlessNicUntranslated, ..Default::default() }; let mut qemu = QemuInstance::boot_with_options(&netcase(), &[], &[], options); let refused = untranslated_nic(); - let gone = "netstack: no NIC driver brought a link in 30 s; this machine has no network, exiting"; + let gone = "netstack: DHCP: no lease as toyos-t14 on any link in"; let mut text = qemu.boot_log().to_string(); qemu::await_guest(&mut qemu, &mut text, "the supervisor's refusal", |t| t.contains(&refused)).map_err(|e| format!("{e}\n{text}"))?; - // A quiet guest, for netstack's bound on its first link: a ceiling past it. + // A quiet guest, for netstack's bound on its first lease: a ceiling past it. if !text.contains(gone) { text.push_str(&qemu.drain_until(std::time::Duration::from_secs(90), |l| l.contains(gone))); } if !text.contains(gone) { - return Err(format!("netstack did not leave a machine whose NIC has no driver\n{text}")); + return Err(format!("netstack did not say a machine whose NIC has no driver has no address\n{text}")); } let log = Serial::named("boot console", text); log.must_be_clean()?; diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 0e18fd74c0d..07c1ee86a0e 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -701,6 +701,9 @@ pub enum Profile { /// [`Profile::Headless`] with an `e1000e`, QEMU's 82574L, where its virtio /// NIC was: a NIC the image's rows name no machine for. HeadlessE1000e, + /// [`Profile::Headless`] with two virtio NICs, each on a user network of + /// its own: [`TWO_NICS`]. + HeadlessTwoNics, /// [`Profile::Headless`] with the kernel's virtio console created without /// `iommu_platform=on`: a device the kernel itself must tell physical /// addresses. @@ -809,6 +812,7 @@ impl Profile { | Self::HeadlessNicBehindPort | Self::HeadlessNicUntranslated | Self::HeadlessE1000e + | Self::HeadlessTwoNics | Self::HeadlessConsoleUntranslated | Self::HeadlessTwoEdu | Self::Metal @@ -938,8 +942,15 @@ enum Nic { /// QEMU's `e1000e`, which the unit decodes as it does every function /// that is not virtio. E1000e, + /// Two virtio NICs: [`TWO_NICS`]. + VirtioPair, } +/// [`Profile::HeadlessTwoNics`]'s NICs, in the order QEMU creates them: each +/// device's id, which QMP's `set_link` names, and its user network, which +/// QEMU's host is `.2` of. The first is QEMU's default network. +pub const TWO_NICS: [(&str, &str); 2] = [("nic0", "10.0.2.0/24"), ("nic1", "10.0.3.0/24")]; + /// Everything a profile decides about the machine, in one table. A new /// variant answers every question here or does not compile — which `self != /// Profile::Metal` did the opposite of: it handed anything that was not @@ -1142,6 +1153,7 @@ impl Profile { Self::HeadlessNicBehindPort => Shape { nic: Nic::BehindPort, ..Self::Headless.shape() }, Self::HeadlessNicUntranslated => Shape { nic: Nic::Untranslated, ..Self::Headless.shape() }, Self::HeadlessE1000e => Shape { nic: Nic::E1000e, ..Self::Headless.shape() }, + Self::HeadlessTwoNics => Shape { nic: Nic::VirtioPair, ..Self::Headless.shape() }, Self::HeadlessConsoleUntranslated => Shape { console_bypasses: true, ..Self::Headless.shape() }, Self::HeadlessTwoEdu => Shape { probes: Probes::Edus, ..Self::Headless.shape() }, Self::HeadlessUsbSpare => Shape { @@ -2175,6 +2187,21 @@ fn shutdown_reason(bytes: &[u8]) -> Option { Some(value.to_string()) } +/// An open QMP connection for a NIC's link: the carrier its guest driver +/// reads. +pub struct QmpLink(Qmp); + +impl QmpLink { + pub fn open(socket: &Path) -> Self { + Self(Qmp::connect(socket)) + } + + /// The NIC `id`'s link up or down, as its driver will read it. + pub fn set(&mut self, id: &str, up: bool) { + self.0.execute(&format!("{{\"execute\":\"set_link\",\"arguments\":{{\"name\":\"{id}\",\"up\":{up}}}}}")); + } +} + /// An open QMP connection to QEMU's human monitor, for the questions QMP has /// no command of its own for. pub struct QmpMonitor(Qmp); @@ -2514,6 +2541,15 @@ fn qemu_command( Nic::E1000e => { qemu.arg("-netdev").arg(&netdev).arg("-device").arg("e1000e,netdev=net0,romfile="); } + Nic::VirtioPair => { + for (n, (id, net)) in TWO_NICS.iter().enumerate() { + let netdev = if n == 0 { netdev.clone() } else { format!("user,id=net{n},net={net}") }; + qemu.arg("-netdev").arg(&netdev).arg("-device").arg(format!( + "virtio-net-pci-non-transitional,netdev=net{n},id={id},mac=52:54:00:12:34:5{}{platform}", + 6 + n + )); + } + } Nic::BehindPort => { qemu.arg("-device").arg("pcie-root-port,id=nicport,chassis=9"); qemu.arg("-netdev").arg(&netdev).arg("-device").arg(format!( diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index c15d6a9a55c..73a49c9221e 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -475,9 +475,9 @@ pub fn persistent_dir() -> Option<(String, String, String)> { } /// What a transfer row reports of the stack beneath it, as netstack's `inspect` answers it: [tcp]'s -/// and the card's counters. Those a card does not report are left out. +/// counters, and the cards' summed over netstack's links. Those no card reports are left out. pub fn net_counters() -> Result, String> { - const READ: [&str; 33] = [ + const READ: [&str; 25] = [ "net.tcp.sent.bytes", "net.tcp.retransmit.bytes", "net.tcp.rto", @@ -503,23 +503,30 @@ pub fn net_counters() -> Result, String> { "net.tcp.hystart.resumed", "net.tcp.hystart.exits", "net.tcp.hystart.exit.cwnd.bytes", - "net.transmit.full", - "net.transmit.wake_armed", - "net.transmit.wake_taken", - "net.transmit.full.ns", - "net.transmit.unreported", - "net.errors.missed", - "net.wire.sent", - "net.wire.received", + ]; + /// Each link's, under `net.link..`. + const CARD: [&str; 8] = [ + "transmit.full", + "transmit.wake_armed", + "transmit.wake_taken", + "transmit.full.ns", + "transmit.unreported", + "errors.missed", + "wire.sent", + "wire.received", ]; let snap = inspect::ask(toyos_inspect::NET)?; - Ok(READ - .iter() - .filter_map(|key| match snap.get(*key) { - Some(toyos_inspect::Value::U64(v)) => Some((key.to_string(), *v)), - _ => None, - }) - .collect()) + let number = |key: &str| match snap.get(key) { + Some(toyos_inspect::Value::U64(v)) => Some(*v), + _ => None, + }; + let links = number("net.links").unwrap_or(0); + let tcp = READ.iter().filter_map(|key| number(key).map(|v| (key.to_string(), v))); + let cards = CARD.iter().filter_map(|key| { + let each: Vec = (0..links).filter_map(|n| number(&format!("net.link.{n}.{key}"))).collect(); + (!each.is_empty()).then(|| (format!("net.{key}"), each.iter().sum())) + }); + Ok(tcp.chain(cards).collect()) } /// What a row's own connections say of themselves: nothing on ToyOS, whose @@ -538,9 +545,25 @@ impl StreamLog { } } -/// What carries a connection on ToyOS: netstack drives one card, Ethernet. -pub fn link_of(_local: std::net::IpAddr) -> Result<(osbench_lan::link::Link, String), String> { - Ok((osbench_lan::link::Link::Wired, "wired netstack's one Ethernet card".into())) +/// What carries a connection on ToyOS: the netstack link whose lease holds its local address. +pub fn link_of(local: std::net::IpAddr) -> Result<(osbench_lan::link::Link, String), String> { + let snap = inspect::ask(toyos_inspect::NET)?; + let text = |key: String| match snap.get(&key) { + Some(toyos_inspect::Value::Text(t)) => Some(t.clone()), + _ => None, + }; + let links = match snap.get("net.links") { + Some(toyos_inspect::Value::U64(n)) => *n, + _ => 0, + }; + let held = (0..links).find(|n| text(format!("net.link.{n}.lease.address")).is_some_and(|a| a.split('/').next() == Some(&local.to_string()))); + let Some(n) = held else { return Err(format!("no netstack link holds {local}")) }; + let driver = text(format!("net.link.{n}.driver")).unwrap_or_default(); + match text(format!("net.link.{n}.medium")).as_deref() { + Some("wired") => Ok((osbench_lan::link::Link::Wired, format!("netstack's wired link, {driver}"))), + Some("wireless") => Ok((osbench_lan::link::Link::Wifi, format!("netstack's wireless link, {driver}"))), + other => Err(format!("netstack's link {n} is carried over {other:?}")), + } } /// ToyOS reads no temperature: the kernel counts none diff --git a/tests/guest/rust-tests/Cargo.toml b/tests/guest/rust-tests/Cargo.toml index b2def872cd4..e9ff628ce37 100644 --- a/tests/guest/rust-tests/Cargo.toml +++ b/tests/guest/rust-tests/Cargo.toml @@ -11,6 +11,8 @@ toyos = { path = "../../../sdk/toyos", features = ["std"] } toyos-window = { path = "../../../sdk/window" } toyos-tco = { path = "../../../lib/tco" } toyos-inspect = { path = "../../../lib/inspect" } +# The reader's question to netstack, for `link_failover`'s per-link counts. +inspect = { path = "../../../system/inspect" } # A login session opened by a job, for `app_view`'s grants. toyos-session = { path = "../../../lib/session" } # The log as logkeeper serves it, and the asking for it, for `counters_metal`'s diff --git a/tests/guest/rust-tests/src/bin/link_failover.rs b/tests/guest/rust-tests/src/bin/link_failover.rs new file mode 100644 index 00000000000..3714b667a40 --- /dev/null +++ b/tests/guest/rust-tests/src/bin/link_failover.rs @@ -0,0 +1,136 @@ +//! Traffic fails over to the second of two links when the first goes down, +//! and returns when it comes back. +//! +//! On a machine with two virtio NICs, each on a QEMU user network of its own, +//! this job waits for both links' leases, then sends datagrams to an address +//! off both networks: they leave by the link netstack's default route takes, +//! which netstack's own count of each link's frames (`inspect net.link..sent`) +//! names. It asks the harness, on a stream over the other link, to set the +//! first link's NIC down; the next datagrams leave by the other link and none +//! by the first. Asked to set it up again, the datagrams return to it. + +#[path = "../served_log.rs"] +mod served_log; + +use std::collections::BTreeMap; +use std::io::{BufRead, BufReader, Write}; +use std::net::{Ipv4Addr, SocketAddr, TcpStream, UdpSocket}; +use std::time::{Duration, Instant}; + +use toyos_inspect::Value; + +/// Off both user networks: RFC 5737's TEST-NET-1, which leaves by a default +/// route and answers nothing. +const FAR: (Ipv4Addr, u16) = (Ipv4Addr::new(192, 0, 2, 1), 9); + +/// Datagrams a round sends: more than the frames a link carries of its own +/// in the time a round takes, so a round's link is the one past it. +const ROUND: u64 = 64; + +/// A liveness ceiling on each wait, never a timing. +const BOUND: Duration = Duration::from_secs(90); + +/// netstack's snapshot. +fn snapshot() -> BTreeMap { + inspect::ask(toyos_inspect::NET).unwrap_or_else(|why| panic!("link_failover: netstack's snapshot: {why}")) +} + +fn text<'a>(snap: &'a BTreeMap, key: &str) -> Option<&'a str> { + match snap.get(key) { + Some(Value::Text(text)) => Some(text), + _ => None, + } +} + +/// The frames netstack has handed link `n`. +fn sent(snap: &BTreeMap, n: usize) -> u64 { + match snap.get(&format!("net.link.{n}.sent")) { + Some(Value::U64(sent)) => *sent, + other => panic!("link_failover: link {n}'s sent count is {other:?}"), + } +} + +/// Polls netstack's snapshot until `done` holds of it. +fn until(what: &str, mut done: impl FnMut(&BTreeMap) -> bool) -> BTreeMap { + let by = Instant::now() + BOUND; + loop { + let snap = snapshot(); + if done(&snap) { + return snap; + } + assert!(Instant::now() < by, "link_failover: {what} within {BOUND:?}; netstack says {snap:?}"); + std::thread::yield_now(); + } +} + +/// A round of datagrams, and each link's frames from before it to once one +/// link has carried the round. +fn round(socket: &UdpSocket) -> [u64; 2] { + let before = snapshot(); + let before = [sent(&before, 0), sent(&before, 1)]; + for n in 0..ROUND { + socket.send_to(&n.to_le_bytes(), FAR).expect("link_failover: a datagram queued"); + } + let after = until("a link carrying the round", |snap| (0..2).any(|n| sent(snap, n) >= before[n] + ROUND)); + [sent(&after, 0) - before[0], sent(&after, 1) - before[1]] +} + +/// The link carrying a round, and the other. +fn carried(by: [u64; 2]) -> (usize, usize) { + if by[0] >= ROUND { (0, 1) } else { (1, 0) } +} + +fn main() { + let port: u16 = std::env::args() + .nth(1) + .and_then(|port| port.parse().ok()) + .expect("link_failover: the harness's port is this job's argument"); + let mut leases = 0; + served_log::Log::open().until("both links' leases", BOUND, |line| { + leases += usize::from(line.contains("netstack: DHCP: lease ")); + leases == 2 + }); + let snap = snapshot(); + assert_eq!(snap.get("net.links"), Some(&Value::U64(2)), "link_failover: two links"); + // Each link's network, by its lease's address: QEMU's host is `.2` of it. + let network = |n: usize| { + let address = text(&snap, &format!("net.link.{n}.lease.address")).unwrap_or_else(|| panic!("link_failover: link {n} holds no lease")); + let address: Ipv4Addr = address.split('/').next().and_then(|a| a.parse().ok()).expect("link_failover: an address"); + let [a, b, c, _] = address.octets(); + Ipv4Addr::new(a, b, c, 0) + }; + let networks = [network(0), network(1)]; + + let socket = UdpSocket::bind("0.0.0.0:0").expect("link_failover: a datagram socket"); + let (first, second) = carried(round(&socket)); + println!(" PASS: a round left by link {first}, on {}", networks[first]); + + // The harness, on a stream over the second link: its network's own host. + let [a, b, c, _] = networks[second].octets(); + let mut control = TcpStream::connect(SocketAddr::from((Ipv4Addr::new(a, b, c, 2), port))).expect("link_failover: a stream to the harness over the second link"); + control.set_read_timeout(Some(BOUND)).expect("link_failover: a read timeout"); + let mut answers = BufReader::new(control.try_clone().expect("link_failover: the stream's reader")); + let mut ask = |word: &str| { + writeln!(control, "{word} {}", networks[first]).expect("link_failover: the ask"); + let mut line = String::new(); + answers.read_line(&mut line).expect("link_failover: the harness's answer"); + assert_eq!(line, format!("{word}\n"), "link_failover: the harness answered {line:?}"); + }; + + ask("down"); + until("the first link down", |snap| text(snap, &format!("net.link.{first}.state")) == Some("down")); + let by = round(&socket); + assert_eq!(carried(by).0, second, "link_failover: the round left by {by:?}"); + assert_eq!(by[first], 0, "link_failover: nothing leaves by a link that is down"); + println!(" PASS: with link {first} down, a round left by link {second} and none by link {first}"); + + ask("up"); + until("the first link up", |snap| text(snap, &format!("net.link.{first}.state")) == Some("up")); + // Its lease verified again, the default route is the first link's. + let by = Instant::now() + BOUND; + while carried(round(&socket)).0 != first { + assert!(Instant::now() < by, "link_failover: no round left by link {first} within {BOUND:?} of its return"); + } + println!(" PASS: with link {first} up again, a round left by it"); + println!("link_failover: traffic failed over to the second link and returned to the first"); +} diff --git a/tests/images/benchcase.toml b/tests/images/benchcase.toml index 7e4acdfa46c..40fbc32f588 100644 --- a/tests/images/benchcase.toml +++ b/tests/images/benchcase.toml @@ -36,7 +36,7 @@ receives = ["power"] # netstack over the T14's wired I219, the only card a bench has that ToyOS # drives: `osbench lan` reaches its peer and `osbench internet` the internet # through it. On the AMD bench, which has none, the supervisor starts no -# driver, netstack leaves, and both say so. +# driver, netstack has no address, and both say so. [programs.netstack] service = true serves = ["netstack", "link"] diff --git a/tests/images/metalcase.toml b/tests/images/metalcase.toml index 0e7dbaa0941..073aee6b7b6 100644 --- a/tests/images/metalcase.toml +++ b/tests/images/metalcase.toml @@ -45,8 +45,8 @@ drives = ["pci:04:03:*"] receives = ["sound-output"] syscap = ["rt"] -# netstack over the link a NIC driver brings; on a machine with no virtio NIC -# the supervisor starts no driver, and netstack leaves. +# netstack over every link a NIC driver brings; on a machine with no virtio NIC +# the supervisor starts no driver, and netstack serves with no address. [programs.netstack] service = true serves = ["netstack", "link"] diff --git a/tests/images/netcase.toml b/tests/images/netcase.toml index 19598e0ecec..d0684da33a5 100644 --- a/tests/images/netcase.toml +++ b/tests/images/netcase.toml @@ -2,9 +2,9 @@ # NVMe controller and soundserver's virtio-sound beside it: `iommu_virtio_platform`'s # machine, where every virtio function is a process's claim or a kernel driver's. # -# netstack serves once a NIC driver says hello, and leaves a machine none says -# hello on within 30 s. Every NIC driver is here, so whichever NIC the machine -# has comes up with no row naming it. +# netstack serves every link a NIC driver brings, from the start. Every NIC +# driver is here, so whichever NICs the machine has come up with no row +# naming them. # # test-runner is here for the same reason it is in metalcase: it makes the boot # announce itself. diff --git a/tests/toyos.rs b/tests/toyos.rs index 4dc4052e248..635655f8927 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -269,6 +269,9 @@ const RUST_SKIP: &[&str] = &[ // netstack, which a shared boot runs none of: // `a_client_killed_mid_connect_gives_its_place_back` runs it. "netstack_connect_killed", + // It needs two NICs and a harness that sets one's link down: + // `traffic_leaves_by_the_second_link_while_the_first_is_down` runs it. + "link_failover", // It needs a launcher: `a_launch_gives_each_unsent_stdio_slot_a_null` // runs it on tests/images/launchcase.toml. "launch_unsent_stdio", @@ -676,6 +679,16 @@ const MACHINE_TESTS: &[&str] = &[ // netstack takes to give the address up, or the test is red. x86-64 alone // (`qemu::SUITE_ARCH`). "a_client_killed_mid_connect_gives_its_place_back", + // netstack over two links: the default route leaves by the first, by the + // second while QMP holds the first's NIC down, and by the first again + // when it is up. Two drivers' links, a NIC's carrier and netstack's + // routes cross three processes and two devices, which the node's and the + // shard's host tests of rank, failover and return hold below the drivers. + // QEMU x86-64 (`qemu::SUITE_ARCH`), on two user networks; the datagrams go + // to RFC 5737's TEST-NET-1, which answers nothing, and the harness is the + // peer on the host's loopback, so no public network. Order and counts + // only. + "traffic_leaves_by_the_second_link_while_the_first_is_down", // Login sessions: their owner parents, signs out and ends with each, and // the machine's reboot waits for them until forced, which only processes // in one boot show, as order, completion and counts. The reboot ends the @@ -4703,6 +4716,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "udp_receive_times_out" => udp_receive_times_out(), "a_link_ends_under_a_held_stream_and_comes_back" => a_link_ends_under_a_held_stream_and_comes_back(), "a_client_killed_mid_connect_gives_its_place_back" => a_client_killed_mid_connect_gives_its_place_back(), + "traffic_leaves_by_the_second_link_while_the_first_is_down" => traffic_leaves_by_the_second_link_while_the_first_is_down(), "session_lifecycle" => common::sessionlifecycle::session_lifecycle(), "a_launch_gives_each_unsent_stdio_slot_a_null" => common::launchstdio::a_launch_gives_each_unsent_stdio_slot_a_null(), "a_row_reaches_its_own_state_and_no_other" => common::stateview::a_row_reaches_its_own_state_and_no_other(), @@ -5796,6 +5810,57 @@ fn peer_verdict(heard: std::thread::JoinHandle>) -> String /// answers each line the job writes on one connection, the one before the /// job ends its driver and the one after it starts another, then reads the /// job's FIN and answers it with its own. +/// `test_rs_link_failover` on two virtio NICs: the harness, asked over the +/// second link's network, sets the first link's NIC down and then up. +fn traffic_leaves_by_the_second_link_while_the_first_is_down() -> Result<(), String> { + use std::io::{BufRead, BufReader, Write}; + const JOB: &str = "link_failover"; + /// A liveness ceiling on the job and the peer's waits, never a timing. + const CEILING: Duration = Duration::from_secs(300); + let tests = compile::repo_root().join("tests/guest/rust-tests"); + let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &tests, JOB); + let config = compile::repo_root().join("tests/images/netcase.toml"); + let options = BootOptions { profile: qemu::Profile::HeadlessTwoNics, qmp: true, ..BootOptions::default() }; + let mut qemu = QemuInstance::boot_with_options(&config, &[], &[(JOB.to_string(), bin)], options); + serial::Serial::boot(&qemu).must_be_clean()?; + let peer = std::net::TcpListener::bind("127.0.0.1:0").map_err(|e| format!("no loopback listener: {e}"))?; + let port = peer.local_addr().map_err(|e| format!("the listener's port: {e}"))?.port(); + let socket = qemu.qmp_socket().to_path_buf(); + let heard = std::thread::spawn(move || -> Result<(), String> { + let (conn, _) = peer.accept().map_err(|e| format!("the guest's connect: {e}"))?; + conn.set_read_timeout(Some(CEILING)).map_err(|e| e.to_string())?; + let mut reader = BufReader::new(conn.try_clone().map_err(|e| e.to_string())?); + let mut answer = conn; + let mut link = qemu::QmpLink::open(&socket); + for word in ["down", "up"] { + let mut line = String::new(); + reader.read_line(&mut line).map_err(|e| format!("the job's {word:?} ask: {e}"))?; + let Some((said, network)) = line.trim_end().split_once(' ') else { return Err(format!("the job asked {line:?}")) }; + if said != word { + return Err(format!("the job asked {line:?} where {word:?} was owed")); + } + let Some((id, _)) = qemu::TWO_NICS.iter().find(|(_, net)| net.split('/').next() == Some(network)) else { + return Err(format!("the job named {network:?}, which is neither NIC's network")); + }; + link.set(id, word == "up"); + answer.write_all(format!("{word}\n").as_bytes()).map_err(|e| format!("the answer to {word:?}: {e}"))?; + } + Ok(()) + }); + let result = qemu.run_test(&format!("test_rs_{JOB} {port}"), CEILING); + if let Some(why) = &result.error { + return Err(format!("{why}\n{}\nthe job said:\n{}", peer_verdict(heard), result.stdout)); + } + if result.exit_code != Some(0) { + return Err(format!("the job ended {:?}:\n{}", result.exit_code, result.stdout)); + } + heard.join().map_err(|_| "the peer's thread panicked".to_string())??; + for line in result.stdout.lines().filter(|l| l.contains("PASS: ")) { + eprintln!(" [links] {}", line.trim()); + } + Ok(()) +} + fn a_link_ends_under_a_held_stream_and_comes_back() -> Result<(), String> { use std::io::{BufRead, BufReader, Write}; const JOB: &str = "link_restart"; From 870f7548de9a2df34356a8ed30748d72f6ecc5b1 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 31 Oct 2026 20:11:38 +0100 Subject: [PATCH 4/9] virtio-net reports its link, and the failover test reads it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two-NIC guest test found QMP's set_link reaching nothing: virtio-net accepted no VIRTIO_NET_F_STATUS, so its link was Unreported and netstack took it as up whatever QEMU said. The driver now accepts the feature where the device offers it and reads `status` (§5.1.4) each pass; a change of it is a configuration change (§5.1.4.2), which raises the one vector, so a pass follows it. toyos-virtio's live device reads a 16-bit field of its structure, bounded and aligned as its 32-bit ones (§4.1.3.1). The link word gains UpWithoutSpeed, its free fourth tag: a device that reports its carrier and no speed, as virtio does without VIRTIO_NET_F_SPEED_DUPLEX. inspect reads it as up. The failover job waits on netstack's inspect for both leases: netcase gives its jobs no log port. Co-Authored-By: Claude Opus 5.5 --- lib/link/src/state.rs | 11 ++++++-- lib/virtio/src/pci.rs | 16 ++++++++++- lib/virtio/src/tests.rs | 19 +++++++++++++ system/netstack/src/link.rs | 1 + system/virtio-net/src/nic.rs | 28 +++++++++++++++++-- tests/guest/rust-tests/Cargo.lock | 10 +++++++ .../guest/rust-tests/src/bin/link_failover.rs | 11 ++------ 7 files changed, 81 insertions(+), 15 deletions(-) diff --git a/lib/link/src/state.rs b/lib/link/src/state.rs index fb33c6c7b1f..ad3333850be 100644 --- a/lib/link/src/state.rs +++ b/lib/link/src/state.rs @@ -8,11 +8,15 @@ pub enum Link { /// The device reports no link at all, as virtio's network device without /// `VIRTIO_NET_F_STATUS` does; the stack takes it as up. Unreported, + /// The device reports its link up and no speed, as virtio's network + /// device with `VIRTIO_NET_F_STATUS` and without `VIRTIO_NET_F_SPEED_DUPLEX`. + UpWithoutSpeed, } const DOWN: u32 = 0; const UNREPORTED: u32 = 1; const UP: u32 = 2; +const UP_WITHOUT_SPEED: u32 = 3; const FULL_DUPLEX: u32 = 1 << 2; const MBPS_SHIFT: u32 = 8; @@ -27,6 +31,7 @@ impl Link { match self { Self::Down => DOWN, Self::Unreported => UNREPORTED, + Self::UpWithoutSpeed => UP_WITHOUT_SPEED, Self::Up { mbps, full_duplex } => { let mbps = mbps.clamp(1, u32::MAX >> MBPS_SHIFT); UP | if full_duplex { FULL_DUPLEX } else { 0 } | mbps << MBPS_SHIFT @@ -41,6 +46,7 @@ impl Link { match word & 0b11 { DOWN if word == DOWN => Some(Self::Down), UNREPORTED if word == UNREPORTED => Some(Self::Unreported), + UP_WITHOUT_SPEED if word == UP_WITHOUT_SPEED => Some(Self::UpWithoutSpeed), UP if reserved == 0 && mbps > 0 => Some(Self::Up { mbps, full_duplex: word & FULL_DUPLEX != 0 }), _ => None, } @@ -175,6 +181,7 @@ mod tests { for link in [ Link::Down, Link::Unreported, + Link::UpWithoutSpeed, Link::Up { mbps: 10, full_duplex: false }, Link::Up { mbps: 1000, full_duplex: true }, Link::Up { mbps: u32::MAX >> 8, full_duplex: true }, @@ -185,10 +192,10 @@ mod tests { } /// Every word no driver writes is refused: an up link of no speed, a - /// reserved bit, a down or unreported link carrying fields, a fourth tag. + /// reserved bit, a down, unreported or speedless link carrying fields. #[test] fn a_word_no_driver_writes_names_no_link() { - for word in [UP, UP | FULL_DUPLEX, UP | 1 << 3 | 1 << 8, DOWN | 1 << 8, UNREPORTED | FULL_DUPLEX, 3, 3 | 1 << 8, u32::MAX] { + for word in [UP, UP | FULL_DUPLEX, UP | 1 << 3 | 1 << 8, DOWN | 1 << 8, UNREPORTED | FULL_DUPLEX, UP_WITHOUT_SPEED | 1 << 8, UP_WITHOUT_SPEED | FULL_DUPLEX, u32::MAX] { assert_eq!(Link::decode(word), None, "{word:#x}"); } } diff --git a/lib/virtio/src/pci.rs b/lib/virtio/src/pci.rs index 0cb974c7d25..57cf2081c7e 100644 --- a/lib/virtio/src/pci.rs +++ b/lib/virtio/src/pci.rs @@ -365,8 +365,15 @@ impl Wires { /// Where in the BAR the 32-bit device-specific field `at` is, read or /// written 32 bits wide as §4.1.3.1 has one. fn device32(&mut self, at: usize) -> Result { + self.device_field(at, 4) + } + + /// Where the `width`-byte field `at` bytes into the device-specific + /// structure is, refusing one that is not a whole aligned field of it + /// (§4.1.3.1). + fn device_field(&mut self, at: usize, width: usize) -> Result { let Region { at: base, bytes } = self.device; - if at.checked_add(4).is_none_or(|end| end > bytes) || !at.is_multiple_of(4) { + if at.checked_add(width).is_none_or(|end| end > bytes) || !at.is_multiple_of(width) { return Err(self.refuse(Refusal::PastDeviceConfig { at, bytes })); } Ok(base + at) @@ -576,6 +583,13 @@ impl Live { Ok(self.wires.regs.read32(at)) } + /// The 16-bit field `at` bytes into the device-specific structure, read + /// 16 bits wide (§4.1.3.1). A refusal sets `FAILED`. + pub fn device_read16(&mut self, at: usize) -> Result { + let at = self.wires.device_field(at, 2)?; + Ok(self.wires.regs.read16(at)) + } + /// Write the 32-bit field `at` bytes into the device-specific structure. pub fn device_write32(&mut self, at: usize, value: u32) -> Result<(), Refusal> { let at = self.wires.device32(at)?; diff --git a/lib/virtio/src/tests.rs b/lib/virtio/src/tests.rs index c19d0c0881c..c6628770724 100644 --- a/lib/virtio/src/tests.rs +++ b/lib/virtio/src/tests.rs @@ -482,6 +482,25 @@ fn a_live_device_reads_and_writes_its_own_fields_inside_the_structure() { } } +/// §4.1.3.1: a 16-bit field is read 16 bits wide, and one that is not a whole +/// aligned word of the structure is refused, with `FAILED`, and not read. +#[test] +fn a_live_devices_16_bit_field_is_read_16_bits_wide_and_only_inside_the_structure() { + let machine = Machine::new(); + let (mut device, _queue) = live(&machine); + machine.forget_trace(); + assert_eq!(device.device_read16(4), Ok(0x5634)); + assert_eq!(machine.trace(), [Event::DeviceConfig { at: 4 }]); + for at in [6, 5, usize::MAX] { + let machine = Machine::new(); + let (mut device, _queue) = live(&machine); + machine.forget_trace(); + assert_eq!(device.device_read16(at), Err(Refusal::PastDeviceConfig { at, bytes: 6 })); + let failed = status::ACKNOWLEDGE | status::DRIVER | status::FEATURES_OK | status::DRIVER_OK | status::FAILED; + assert_eq!(machine.trace(), [Event::Status(failed)]); + } +} + // --- PCI 3.0 §6.7: the capability list --- /// A function's configuration space: 256 bytes, and the offsets a claim diff --git a/system/netstack/src/link.rs b/system/netstack/src/link.rs index 89f6eb25325..30fdb000755 100644 --- a/system/netstack/src/link.rs +++ b/system/netstack/src/link.rs @@ -265,6 +265,7 @@ impl Session { match self.said { Link::Down => snap.put(&format!("{at}.state"), "down"), Link::Unreported => snap.put(&format!("{at}.state"), "unreported"), + Link::UpWithoutSpeed => snap.put(&format!("{at}.state"), "up"), Link::Up { mbps, full_duplex } => { snap.put(&format!("{at}.state"), "up"); snap.put(&format!("{at}.speed_mbps"), mbps); diff --git a/system/virtio-net/src/nic.rs b/system/virtio-net/src/nic.rs index 469be896e22..8e42c2cb53c 100644 --- a/system/virtio-net/src/nic.rs +++ b/system/virtio-net/src/nic.rs @@ -34,6 +34,13 @@ use toyos_virtio::queue::{avail_bytes, desc_bytes, Buffer, Parts, Published, Use /// §5.1.3: the device has a MAC address of its own to read. const VIRTIO_NET_F_MAC: u64 = 1 << 5; +/// §5.1.3: the device reports its link in `status`, and a change of it is a +/// configuration change (§5.1.4.2). +const VIRTIO_NET_F_STATUS: u64 = 1 << 16; +/// §5.1.4: `status`, after the six bytes of `mac`; its bit 0 is +/// `VIRTIO_NET_S_LINK_UP`. +const STATUS_AT: usize = 6; +const VIRTIO_NET_S_LINK_UP: u16 = 1; /// The one MSI-X table entry the kernel programs. const MSIX_ENTRY: u16 = 0; @@ -101,6 +108,8 @@ pub struct VirtioNet { rx: Virtqueue, tx: TxQueue, mac: [u8; 6], + /// The device accepted `VIRTIO_NET_F_STATUS`, and reports its link. + status: bool, } impl VirtioNet { @@ -108,7 +117,8 @@ impl VirtioNet { /// feature negotiation, both queues and their vectors in the order /// `toyos-virtio`'s types fix, which is virtio 1.2 §3.1.1's. pub fn open(dev: PciDev) -> Result { - let Negotiated { setup, mapping, features } = negotiate(&dev, VIRTIO_NET_F_MAC)?; + let Negotiated { setup, mapping, features } = negotiate(&dev, VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS)?; + let status = setup.features() & VIRTIO_NET_F_STATUS != 0; say!("virtio-net: {features}"); let (grant, region) = Grant::alloc(&dev, GRANT_BYTES)?; @@ -137,6 +147,7 @@ impl VirtioNet { rx, tx, mac, + status, }; // Every receive buffer posted before a frame can arrive. @@ -195,12 +206,23 @@ impl toyos_link::driver::Nic for VirtioNet { /// kernel saying this function is no longer this process's, and this /// dies where it can be read. /// - /// The link is unreported: this driver accepts no `VIRTIO_NET_F_STATUS`. + /// The link is `status`'s where the device accepted + /// `VIRTIO_NET_F_STATUS`, read each pass: a change of it raises the + /// configuration vector, the one interrupt, so a pass follows it. A + /// device without the feature reports none. fn begin_pass(&mut self) -> Link { match self.dev.irq() { - Ok(_) | Err(SyscallError::WouldBlock) => Link::Unreported, + Ok(_) | Err(SyscallError::WouldBlock) => {} Err(why) => panic!("virtio-net: this NIC cannot be driven on — {why:?}"), } + if !self.status { + return Link::Unreported; + } + match self.device.device_read16(STATUS_AT) { + Ok(word) if word & VIRTIO_NET_S_LINK_UP != 0 => Link::UpWithoutSpeed, + Ok(_) => Link::Down, + Err(why) => panic!("virtio-net: its link status is not where §5.1.4 puts it — {why}"), + } } /// A buffer is posted again once its frame was taken: one never given diff --git a/tests/guest/rust-tests/Cargo.lock b/tests/guest/rust-tests/Cargo.lock index cf129ddd534..2d140087f04 100644 --- a/tests/guest/rust-tests/Cargo.lock +++ b/tests/guest/rust-tests/Cargo.lock @@ -296,6 +296,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inspect" +version = "0.1.0" +dependencies = [ + "toyos", + "toyos-abi", + "toyos-inspect", +] + [[package]] name = "jni" version = "0.22.4" @@ -980,6 +989,7 @@ dependencies = [ "acpiserver-api", "cpal", "getrandom", + "inspect", "libloading", "logkeeper-api", "memmap2", diff --git a/tests/guest/rust-tests/src/bin/link_failover.rs b/tests/guest/rust-tests/src/bin/link_failover.rs index 3714b667a40..da1b47d7165 100644 --- a/tests/guest/rust-tests/src/bin/link_failover.rs +++ b/tests/guest/rust-tests/src/bin/link_failover.rs @@ -9,9 +9,6 @@ //! first link's NIC down; the next datagrams leave by the other link and none //! by the first. Asked to set it up again, the datagrams return to it. -#[path = "../served_log.rs"] -mod served_log; - use std::collections::BTreeMap; use std::io::{BufRead, BufReader, Write}; use std::net::{Ipv4Addr, SocketAddr, TcpStream, UdpSocket}; @@ -85,13 +82,9 @@ fn main() { .nth(1) .and_then(|port| port.parse().ok()) .expect("link_failover: the harness's port is this job's argument"); - let mut leases = 0; - served_log::Log::open().until("both links' leases", BOUND, |line| { - leases += usize::from(line.contains("netstack: DHCP: lease ")); - leases == 2 + let snap = until("both links' leases", |snap| { + snap.get("net.links") == Some(&Value::U64(2)) && (0..2).all(|n| snap.get(&format!("net.link.{n}.lease.held")) == Some(&Value::Bool(true))) }); - let snap = snapshot(); - assert_eq!(snap.get("net.links"), Some(&Value::U64(2)), "link_failover: two links"); // Each link's network, by its lease's address: QEMU's host is `.2` of it. let network = |n: usize| { let address = text(&snap, &format!("net.link.{n}.lease.address")).unwrap_or_else(|| panic!("link_failover: link {n} holds no lease")); From 180db88c9247f9fc6b1c483fbd6faa73240cc27d Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 31 Oct 2026 20:28:57 +0100 Subject: [PATCH 5/9] sdk: the C interface carries eight stream refusals, AddressGone among them files.rs round-trips every stream refusal through the C interface; its count moves from seven to eight with P5's AddressGone. Co-Authored-By: Claude Opus 5.5 --- sdk/toyos/src/files.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/toyos/src/files.rs b/sdk/toyos/src/files.rs index d2d44b1b6fc..6aabdd02d30 100644 --- a/sdk/toyos/src/files.rs +++ b/sdk/toyos/src/files.rs @@ -537,7 +537,7 @@ mod tests { every.extend((0..=255u8).filter_map(fs::Error::from_byte).map(Error::Served)); let streams: alloc::vec::Vec = (0..=255u8).filter_map(crate::net::stream::Refused::from_byte).map(Error::Stream).collect(); - assert_eq!(streams.len(), 7, "every stream refusal but a foreign handle's"); + assert_eq!(streams.len(), 8, "every stream refusal but a foreign handle's"); every.extend(streams); assert!(every.len() > 40, "only {} errors", every.len()); for e in every { From 086291ad243fd73b666b8b5d7200ca4a2c9ef78c Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 00:15:23 +0100 Subject: [PATCH 6/9] review r1 of #1122: next hops named by their link, a stuck link starves no other, one LAN on two links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A next hop is its link's: TCP's reachability advice carries the connection's local address and ip advises the flow's own link's entry (Source::Bound); a UDP datagram waits on and is failed by its next hop on its interface, so one link's dead gateway drops no datagram waiting for the same address on another. - ip's control queue bounds each interface's share: a link that takes nothing fills only its own, and every other link's ARP, ICMP, probes and announcements still queue and leave. - One LAN on two links (RFC 6762 §14's bridged laptop): ip already refuses a datagram from any of its interfaces' addresses (`ip.own-source`), so no responder hears this machine's own mDNS; name.rs says so. A server's reply addressed to another link's MAC is that link's, and the node drops it before this link's client logs it (`node.dhcp-other-link`; toyos_dhcp::addressed_to reads the reply's chaddr). A host test boots the bridged case. - The asked link is sticky: with no link up, lookups keep asking the link they asked; tested whichever link goes down last. - Serving before a link made comments false: the ULP row gates on the lease, netstack_answers goes, sshserver's and netstack_gone_mid_bind's comments say what is true; a detached doc comment is reattached. - Each link's DHCP client has its own refusal log; link lines say `link `; drivers' pending hellos are capped at the link cap; the two-NIC profile's networks are restrict=on, the harness reached through each network's guestfwd `.100:7`. - issues/two-links-leased-one-address-the-second-holds-none.md records the two networks that lease one address. Co-Authored-By: Claude Opus 5.5 --- ...eased-one-address-the-second-holds-none.md | 27 +++ system/netstack/dhcp/src/lib.rs | 1 + system/netstack/dhcp/src/message.rs | 10 ++ system/netstack/dhcp/tests/parse.rs | 15 ++ system/netstack/ip/src/config.rs | 8 +- system/netstack/ip/src/egress.rs | 74 ++++---- system/netstack/ip/tests/acd.rs | 9 +- system/netstack/ip/tests/clk.rs | 2 +- system/netstack/ip/tests/common/mod.rs | 4 +- system/netstack/ip/tests/nud.rs | 36 ++-- system/netstack/ip/tests/rank.rs | 88 ++++++++++ system/netstack/node/src/lease.rs | 32 +++- system/netstack/node/src/lib.rs | 54 ++++-- system/netstack/node/src/name.rs | 4 + system/netstack/node/tests/links.rs | 116 +++++++++++-- system/netstack/shard/Cargo.toml | 2 +- system/netstack/shard/src/lib.rs | 32 ++-- system/netstack/shard/tcp/src/conn.rs | 4 +- system/netstack/shard/tcp/src/counters.rs | 10 +- system/netstack/shard/tcp/src/lib.rs | 9 +- system/netstack/shard/tcp/src/open.rs | 2 +- system/netstack/shard/tcp/tests/modern.rs | 2 +- system/netstack/shard/tcp/tests/timers.rs | 10 +- system/netstack/shard/tests/links.rs | 164 ++++++++++++++++++ system/netstack/src/main.rs | 22 ++- system/netstack/udp/src/lib.rs | 33 ++-- system/netstack/udp/tests/common/mod.rs | 6 +- system/sshserver/src/main.rs | 5 +- tests/common/qemu.rs | 19 +- .../guest/rust-tests/src/bin/link_failover.rs | 16 +- .../rust-tests/src/bin/netstack_answers.rs | 20 --- .../src/bin/netstack_gone_mid_bind.rs | 8 +- tests/toyos.rs | 44 +++-- 33 files changed, 655 insertions(+), 233 deletions(-) create mode 100644 issues/two-links-leased-one-address-the-second-holds-none.md delete mode 100644 tests/guest/rust-tests/src/bin/netstack_answers.rs diff --git a/issues/two-links-leased-one-address-the-second-holds-none.md b/issues/two-links-leased-one-address-the-second-holds-none.md new file mode 100644 index 00000000000..17f3e26da1a --- /dev/null +++ b/issues/two-links-leased-one-address-the-second-holds-none.md @@ -0,0 +1,27 @@ +--- +status: open +kind: defect +opened: 2026-11-01 +--- + +# Two links leased one address: the second never holds one + +netstack runs every link a driver brings (`issues/there-is-no-wifi.md`, P5), +and [ip] keeps an address unique across its interfaces: a second interface +adding an address the first holds is refused `addr.duplicate`. Two +separate networks can lease the same address, two home routers each handing +out 192.168.1.10, and then the second link's probe is refused, its client +counts `node.address-refused` and starts its exchange over, and the same +server offers it the same address again: the second link never holds an +address, its client asks for the boot, and its DHCP lines repeat. + +Nothing breaks the first link, and the uniqueness rule is what keeps +`toyos-net-tcp`'s `lose_address` and the strong host model's source +selection sound: one address names one interface. + +**Owner**: the orchestrator. + +**Exit**: the second link's client declines the address (RFC 2131 §3.1, a +DECLINE) and asks for another, as it would for one ACD found in use, so that +the second link leases a different address where its server has one; with a +host test of two links offered one address. diff --git a/system/netstack/dhcp/src/lib.rs b/system/netstack/dhcp/src/lib.rs index f3790c69f61..70d17fd55e9 100644 --- a/system/netstack/dhcp/src/lib.rs +++ b/system/netstack/dhcp/src/lib.rs @@ -38,6 +38,7 @@ use toyos_net_wire::addr::{is_host, Cidr}; use toyos_net_wire::ethernet::{IndividualMac, MacAddr}; use toyos_net_wire::Instant; +pub use message::addressed_to; use message::{Build, Kind, Reply, ReplyKind}; toyos_net_wire::counters! { diff --git a/system/netstack/dhcp/src/message.rs b/system/netstack/dhcp/src/message.rs index 5ed8b6f8311..11e90557c27 100644 --- a/system/netstack/dhcp/src/message.rs +++ b/system/netstack/dhcp/src/message.rs @@ -153,6 +153,16 @@ fn addresses(bytes: &[u8]) -> Vec { bytes.as_chunks::<4>().0.iter().map(|&a| Ipv4Addr::from(a)).collect() } +/// The client hardware address a server's reply names (RFC 2131 §2), where it is a reply over +/// Ethernet: which of a host's links' clients it is for. Nothing else of it is read. +pub fn addressed_to(bytes: &[u8]) -> Option { + let (&[op, htype, hlen, ..], _) = bytes.split_first_chunk::<4>()?; + if (op, htype, hlen) != (2, 1, 6) { + return None; + } + bytes.get(28..34)?.first_chunk::<6>().map(|&mac| MacAddr(mac)) +} + impl Reply { pub fn parse(bytes: &[u8], mac: MacAddr) -> Result { let (fixed, options) = bytes.split_at_checked(OPTIONS_AT).ok_or(Counter::Truncated)?; diff --git a/system/netstack/dhcp/tests/parse.rs b/system/netstack/dhcp/tests/parse.rs index 7bb962af808..75f6ec492ea 100644 --- a/system/netstack/dhcp/tests/parse.rs +++ b/system/netstack/dhcp/tests/parse.rs @@ -318,3 +318,18 @@ fn s_dhcp_dh_063_a_refused_offer_then_another() { let payload = &sent(&out).payload; assert_eq!(payload[243..249], [54, 4, 192, 0, 2, 253]); } + +/// A reply names the client it is for, which a host with several links reads to tell which +/// link's client is to have it; a message that is no Ethernet reply names none. +#[test] +fn a_reply_names_its_client_and_nothing_else_does() { + let reply = offer(); + assert_eq!(toyos_dhcp::addressed_to(&reply), Some(MAC_A)); + let mut request = reply.clone(); + request[0] = 1; + assert_eq!(toyos_dhcp::addressed_to(&request), None, "a request"); + let mut token_ring = reply.clone(); + token_ring[1] = 6; + assert_eq!(toyos_dhcp::addressed_to(&token_ring), None, "another hardware type"); + assert_eq!(toyos_dhcp::addressed_to(&reply[..33]), None, "short of its chaddr"); +} diff --git a/system/netstack/ip/src/config.rs b/system/netstack/ip/src/config.rs index ff44b67e451..bc0f25138e3 100644 --- a/system/netstack/ip/src/config.rs +++ b/system/netstack/ip/src/config.rs @@ -117,11 +117,11 @@ impl Ip { self.ifaces.get(iface.0).map(|i| i.igmp.mode()) } - /// A transport's advice about its peer `remote`; it lands on the next hop's entry, the - /// gateway's for an off-link peer. - pub fn advise(&mut self, now: Instant, remote: Ipv4Addr, advice: Advice) { + /// A transport's advice about its peer `remote`, whose flow is from `source`; it lands on + /// the next hop's entry on the flow's own link, the gateway's for an off-link peer. + pub fn advise(&mut self, now: Instant, remote: Ipv4Addr, source: Source, advice: Advice) { let now = self.clock(now); - let Ok(route) = route::lookup(&self.ifaces, &self.order, remote, Source::Any, None) else { return }; + let Ok(route) = route::lookup(&self.ifaces, &self.order, remote, source, None) else { return }; let NextHop::Neighbour(next_hop) = route.next_hop else { return }; let Some((i, mut cx)) = self.split(now, route.iface) else { return }; nud::advise(i, &mut cx, next_hop, advice == Advice::Confirmed); diff --git a/system/netstack/ip/src/egress.rs b/system/netstack/ip/src/egress.rs index 84c65ca0586..3476396178f 100644 --- a/system/netstack/ip/src/egress.rs +++ b/system/netstack/ip/src/egress.rs @@ -1,10 +1,11 @@ //! Egress. [ip]'s own frames wait in one FIFO the transmit opportunity drains ahead of the //! data flows: an ARP or IGMP frame is built when it leaves, from the state of that moment, and -//! the timer it feeds starts then. The FIFO holds at most CONTROL_QUEUE frames, at most -//! ECHO_REPLIES of them echo replies; one of [ip]'s own frames past either is dropped and -//! counted, and its producer moves on as if it had left. Three kinds are never dropped: a -//! released datagram's turn, an ACD probe and an ACD announcement. One that finds the FIFO full -//! waits behind it and enters as room appears, ahead of any frame queued later (RFC 5227 +//! the timer it feeds starts then. The FIFO holds at most CONTROL_QUEUE frames of each +//! interface, at most ECHO_REPLIES of them echo replies, so a link that takes nothing holds no +//! other link's frames back; one of [ip]'s own frames past either is dropped and counted, and +//! its producer moves on as if it had left. Three kinds are never dropped: a released datagram's +//! turn, an ACD probe and an ACD announcement. One that finds its interface's share full waits +//! and enters as room appears there, ahead of any frame of that interface queued later (RFC 5227 //! §2.1). A turn holds no datagram: it names the entry whose queue does, and goes with that entry. //! //! Only [ip]'s own ICMP messages wait here for a next hop. A transport's datagram is framed into @@ -76,37 +77,42 @@ pub(crate) struct Turn { #[derive(Debug, Default)] pub(crate) struct Control { items: VecDeque, - echoes: usize, - /// What may not be dropped and found `items` full, in order: empty whenever it has room. + /// What may not be dropped and found its interface's share full, in order: none of an + /// interface whose share has room. waiting: VecDeque, } +fn is_echo(item: &Item) -> bool { + matches!(item, Item::Frame { kind: FrameKind::Echo, .. }) +} + impl Control { - fn full(&self) -> bool { - self.items.len() >= CONTROL_QUEUE + fn queued(&self, iface: IfIndex) -> usize { + self.items.iter().filter(|item| item.iface() == iface).count() + } + + fn full(&self, iface: IfIndex) -> bool { + self.queued(iface) >= CONTROL_QUEUE } /// Queues one of [ip]'s own frames; `false` when it was dropped and counted. pub fn push(&mut self, item: Item, log: &mut Log) -> bool { - let echo = matches!(item, Item::Frame { kind: FrameKind::Echo, .. }); - if echo && self.echoes >= ECHO_REPLIES { + let iface = item.iface(); + if is_echo(&item) && self.items.iter().filter(|i| i.iface() == iface && is_echo(i)).count() >= ECHO_REPLIES { log.count(Counter::IcmpEchoReplyDropped); return false; } - if self.full() { + if self.full(iface) { log.count(Counter::IpControlQueueFull); return false; } - if echo { - self.echoes = self.echoes.saturating_add(1); - } self.items.push_back(item); true } /// Queues a turn, a probe or an announcement, which waits for room rather than being dropped. pub fn hold(&mut self, item: Item) { - if self.full() { + if self.full(item.iface()) { self.waiting.push_back(item); } else { self.items.push_back(item); @@ -117,22 +123,21 @@ impl Control { fn pop(&mut self, mut room: impl FnMut(IfIndex) -> bool) -> Option { let at = self.items.iter().position(|item| room(item.iface()))?; let item = self.items.remove(at)?; - self.forget(&item); self.refill(); Some(item) } - fn forget(&mut self, item: &Item) { - if matches!(item, Item::Frame { kind: FrameKind::Echo, .. }) { - self.echoes = self.echoes.saturating_sub(1); - } - } - - /// Room goes to the oldest item waiting for it. + /// Room goes to the oldest item waiting for its interface's. fn refill(&mut self) { - while !self.full() { - let Some(item) = self.waiting.pop_front() else { return }; - self.items.push_back(item); + let mut at = 0; + while let Some(item) = self.waiting.get(at) { + if self.full(item.iface()) { + at = at.saturating_add(1); + continue; + } + if let Some(item) = self.waiting.remove(at) { + self.items.push_back(item); + } } } @@ -156,11 +161,7 @@ impl Control { /// Drops everything waiting for `iface`, whose link went down. pub fn purge(&mut self, iface: IfIndex) { - let (gone, kept): (Vec, Vec) = self.items.drain(..).partition(|item| item.iface() == iface); - self.items = kept.into(); - for item in &gone { - self.forget(item); - } + self.items.retain(|item| item.iface() != iface); self.waiting.retain(|item| item.iface() != iface); self.refill(); } @@ -213,9 +214,10 @@ pub enum Sent { /// This many bytes of the caller's buffer are the frame, for this interface: one of its /// credit to spend. Frame { len: usize, iface: IfIndex }, - /// Not taken: this next hop's link address is being asked for. [`Event::Resolved`] or - /// [`Event::Failed`] names it when that ends, unless the routes change first. - Pending(Ipv4Addr), + /// Not taken: the link address of this next hop on `iface` is being asked for. + /// [`Event::Resolved`] or [`Event::Failed`] names it when that ends, unless the routes change + /// first. + Pending { iface: IfIndex, next_hop: Ipv4Addr }, /// Not taken: its interface has no room in this opportunity. The hop is its next hop, or /// its destination where it has none on the link. NoRoom { iface: IfIndex, hop: Ipv4Addr }, @@ -367,7 +369,7 @@ impl Ip { let Some((i, mut cx)) = self.split(now, route.iface) else { return Err(Counter::UnknownInterface) }; match hop(i, &mut cx, &route, out.destination, out.broadcast)? { Hop::To(mac) => build(i.mac, mac, &builder, frame).map(|len| Sent::Frame { len, iface: route.iface }).ok_or(Counter::IpExceedsMtu), - Hop::Asked(next_hop) => Ok(Sent::Pending(next_hop)), + Hop::Asked(next_hop) => Ok(Sent::Pending { iface: route.iface, next_hop }), } }); if sent.is_err() { diff --git a/system/netstack/ip/tests/acd.rs b/system/netstack/ip/tests/acd.rs index b6932206e50..e3f0de96207 100644 --- a/system/netstack/ip/tests/acd.rs +++ b/system/netstack/ip/tests/acd.rs @@ -268,13 +268,14 @@ fn s_ip_acd_017_detection_starts_at_the_first_announcement() { assert_eq!(h.count(Counter::AcdDefended), 1); } -/// if1 at 198.51.100.1/24 beside if0, which holds no address. +/// if0 holding 198.51.100.1/24 and nothing else yet, its control queue for the tests to fill +/// with answers to it: each interface's share of the queue is its own. fn beside_if1() -> (H, IfIndex) { let mut h = H::raw(); - let if1 = h.add_if1(); - h.assign(if1, ip4(198, 51, 100, 1), 24, 0); + let if0 = h.if0; + h.assign(if0, ip4(198, 51, 100, 1), 24, 0); h.settle(); - (h, if1) + (h, if0) } #[test] diff --git a/system/netstack/ip/tests/clk.rs b/system/netstack/ip/tests/clk.rs index 4ca0d4b6f9e..628a7823384 100644 --- a/system/netstack/ip/tests/clk.rs +++ b/system/netstack/ip/tests/clk.rs @@ -224,7 +224,7 @@ fn s_ip_clk_010_a_deadline_saturates() { data: b"hi", }; let out = toyos_net_ip::UdpOut { source: A, destination: B, ttl: toyos_net_wire::ipv4::Ttl::DEFAULT, broadcast: false, iface: None, datagram }; - assert_eq!(h.ip.send_udp(late, &usize::MAX, &out, &mut frame), Ok(toyos_net_ip::Sent::Pending(B))); + assert_eq!(h.ip.send_udp(late, &usize::MAX, &out, &mut frame), Ok(toyos_net_ip::Sent::Pending { iface: h.if0, next_hop: B })); assert_eq!(h.ip.transmit(late, usize::MAX, |_, _| {}), 1); assert!(matches!(h.state(B), Some(Nud::Incomplete(_)))); assert_eq!(h.ip.next_deadline(), Some(Instant::from_nanos(u64::MAX))); diff --git a/system/netstack/ip/tests/common/mod.rs b/system/netstack/ip/tests/common/mod.rs index dc745401075..09883196a35 100644 --- a/system/netstack/ip/tests/common/mod.rs +++ b/system/netstack/ip/tests/common/mod.rs @@ -300,7 +300,7 @@ impl H { let reply = eth(MAC_A, m, 0x0806, &arp_packet(2, m, addr, MAC_A, A)); let _ = h.ip.receive(setup, h.if0, &reply); h.settle(); - h.ip.advise(h.clock(), addr, toyos_net_ip::Advice::Confirmed); + h.ip.advise(h.clock(), addr, toyos_net_ip::Source::Any, toyos_net_ip::Advice::Confirmed); assert!(matches!(h.ip.neighbour(h.if0, addr), Some(Nud::Reachable(r)) if r.confirmed() == H::instant(0))); h.rebase(); h @@ -750,7 +750,7 @@ impl H { self.collect(); sent.map(|s| match s { Sent::Frame { len: n, .. } => Some(frame[..n].to_vec()), - Sent::Pending(_) => None, + Sent::Pending { .. } => None, Sent::NoRoom { .. } => unreachable!("every interface has room"), }) } diff --git a/system/netstack/ip/tests/nud.rs b/system/netstack/ip/tests/nud.rs index 60109b1bf81..71a2f0ff0d3 100644 --- a/system/netstack/ip/tests/nud.rs +++ b/system/netstack/ip/tests/nud.rs @@ -123,7 +123,7 @@ fn s_ip_nud_007_positive_advice_moves_the_confirmation() { let reachable = h.ip.reachable_time(h.if0).unwrap(); let ms = |d: std::time::Duration| d.as_millis() as u64; h.at(10_000); - h.ip.advise(h.clock(), REMOTE, Advice::Confirmed); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Confirmed); h.run(ms(reachable) + 1); assert!(h.is_reachable(R), "the original deadline re-arms"); h.run(10_000 + ms(reachable) - 1); @@ -152,7 +152,7 @@ fn s_ip_nud_009_advice_ends_delay() { h.stale(R, MAC_R); h.udp_to(REMOTE).unwrap(); h.at(4_000); - h.ip.advise(h.clock(), REMOTE, Advice::Confirmed); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Confirmed); assert!(h.is_reachable(R)); assert!(h.run(5_000).is_empty()); } @@ -161,7 +161,7 @@ fn s_ip_nud_009_advice_ends_delay() { fn s_ip_nud_010_a_reply_confirms_probe() { let mut h = H::fixture_i_with(R, MAC_R); h.at(100); - h.ip.advise(h.clock(), REMOTE, Advice::Reverify); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(R), Some(Nud::Probe(_)))); h.out(); h.frame(&hex(V_ARP_REPLY_R)); @@ -261,10 +261,10 @@ fn s_ip_nud_015_an_assertion_revives_failed() { fn s_ip_nud_016_requests_are_spaced() { let mut h = H::fixture_i_with(R, MAC_R); h.at(100); - h.ip.advise(h.clock(), REMOTE, Advice::Reverify); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Reverify); let first = h.out(); h.at(600); - h.ip.advise(h.clock(), REMOTE, Advice::Reverify); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Reverify); let second = h.out(); let later = h.run(1_100); assert_eq!(first.iter().map(|o| o.frame.clone()).collect::>(), [hex(V_ARP_POLL_R)]); @@ -279,7 +279,7 @@ fn s_ip_nud_016_requests_are_spaced() { h.reply_from(R, MAC_R); assert!(h.is_reachable(R)); h.at(100); - h.ip.advise(h.clock(), R, Advice::Reverify); + h.ip.advise(h.clock(), R, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(R), Some(Nud::Probe(_)))); assert!(h.out().is_empty(), "100 ms after the last request"); let later = h.run(1_000); @@ -290,7 +290,7 @@ fn s_ip_nud_016_requests_are_spaced() { fn s_ip_nud_017_negative_advice_probes() { let mut h = H::fixture_i_with(R, MAC_R); h.at(100); - h.ip.advise(h.clock(), REMOTE, Advice::Reverify); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(R), Some(Nud::Probe(_)))); assert_eq!(h.out().iter().map(|o| o.frame.clone()).collect::>(), [hex(V_ARP_POLL_R)]); } @@ -299,10 +299,10 @@ fn s_ip_nud_017_negative_advice_probes() { fn s_ip_nud_018_negative_advice_leaves_probe_incomplete_and_failed() { let mut h = H::fixture_i_with(R, MAC_R); h.at(100); - h.ip.advise(h.clock(), REMOTE, Advice::Reverify); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Reverify); h.out(); h.at(200); - h.ip.advise(h.clock(), REMOTE, Advice::Reverify); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(R), Some(Nud::Probe(p)) if p.requests() == 1)); assert!(h.out().is_empty()); @@ -310,12 +310,12 @@ fn s_ip_nud_018_negative_advice_leaves_probe_incomplete_and_failed() { h.ip.set_gateways(h.clock(), h.if0, &[]).unwrap(); h.udp_to(B).unwrap(); h.out(); - h.ip.advise(h.clock(), B, Advice::Reverify); + h.ip.advise(h.clock(), B, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(B), Some(Nud::Incomplete(s)) if s.requests() == 1)); assert!(h.out().is_empty()); h.run(3_000); h.at(3_500); - h.ip.advise(h.clock(), B, Advice::Reverify); + h.ip.advise(h.clock(), B, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(B), Some(Nud::Failed))); assert!(h.out().is_empty()); } @@ -325,14 +325,14 @@ fn s_ip_nud_019_positive_advice_needs_a_mac() { let mut h = H::fixture_i(); h.udp_to(B).unwrap(); h.out(); - h.ip.advise(h.clock(), B, Advice::Confirmed); + h.ip.advise(h.clock(), B, toyos_net_ip::Source::Any, Advice::Confirmed); assert!(matches!(h.state(B), Some(Nud::Incomplete(_)))); h.run(3_000); - h.ip.advise(h.clock(), B, Advice::Confirmed); + h.ip.advise(h.clock(), B, toyos_net_ip::Source::Any, Advice::Confirmed); assert!(matches!(h.state(B), Some(Nud::Failed))); let (mut h, t0) = unreachable_r(); h.at(t0 + 10); - h.ip.advise(h.clock(), REMOTE, Advice::Confirmed); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Confirmed); assert!(h.is_reachable(R)); assert_eq!(h.mac_of(R), Some(MAC_R)); } @@ -341,7 +341,7 @@ fn s_ip_nud_019_positive_advice_needs_a_mac() { fn s_ip_nud_020_advice_lands_on_the_gateway() { let mut h = H::fixture_i_with(R, MAC_R); h.at(100); - h.ip.advise(h.clock(), REMOTE, Advice::Reverify); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(R), Some(Nud::Probe(_)))); assert!(h.state(REMOTE).is_none()); } @@ -404,7 +404,7 @@ fn s_ip_nud_022_a_full_table_evicts_in_order() { let (failed, unreachable, stale) = (neighbour(0), neighbour(1), neighbour(2)); let _ = h.ip.resolve(h.clock(), h.if0, failed, A); reach_wide(&mut h, unreachable); - h.ip.advise(h.clock(), unreachable, Advice::Reverify); + h.ip.advise(h.clock(), unreachable, toyos_net_ip::Source::Any, Advice::Reverify); ask_wide(&mut h, stale); h.run(5_000); assert!(matches!(h.state(failed), Some(Nud::Failed))); @@ -742,7 +742,7 @@ fn b_holds_two_behind_a_full_queue() -> H { /// requests, each lost to the full queue, leave B quiescent UNREACHABLE from 4,000. fn unreachable_b(h: &mut H) { h.frame(&hex(V_ARP_REPLY)); - h.ip.advise(h.clock(), B, Advice::Reverify); + h.ip.advise(h.clock(), B, toyos_net_ip::Source::Any, Advice::Reverify); for t in [1_000, 2_000, 3_000, 4_000] { h.ip.fire(H::instant(t)); } @@ -789,7 +789,7 @@ fn s_ip_nud_030_released_datagrams_keep_their_entry() { h.at(5); h.frame(&hex(V_ARP_REPLY)); h.at(1_000); - h.ip.advise(h.clock(), B, Advice::Reverify); + h.ip.advise(h.clock(), B, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(B), Some(Nud::Probe(_)))); h.frame(ð(MacAddr::BROADCAST, MAC_X, 0x0806, &arp_packet(1, MAC_X, B, MacAddr::ZERO, B))); h.ip.fire(H::instant(601_000)); diff --git a/system/netstack/ip/tests/rank.rs b/system/netstack/ip/tests/rank.rs index fb0da48d8d9..e4e017be836 100644 --- a/system/netstack/ip/tests/rank.rs +++ b/system/netstack/ip/tests/rank.rs @@ -125,3 +125,91 @@ fn a_link_without_room_holds_back_no_other_links_frames() { assert_eq!(ip.transmit(now, 8usize, |iface, _| out.push(iface)), 1); assert_eq!(out, [ifaces[0]], "the first's, still queued"); } + +// ---- one LAN on two links ---- + +/// The wired and wireless links bridged onto one LAN: one prefix, one gateway address. +const LAN_GATEWAY: Ipv4Addr = Ipv4Addr::new(192, 168, 1, 1); +const LAN_WIRED: Net = Net { mac: MacAddr([2, 0, 0, 0, 4, 4]), addr: Ipv4Addr::new(192, 168, 1, 10), gateway: LAN_GATEWAY }; +const LAN_WIRELESS: Net = Net { mac: MacAddr([2, 0, 0, 0, 5, 5]), addr: Ipv4Addr::new(192, 168, 1, 20), gateway: LAN_GATEWAY }; +const GATEWAY_MAC: MacAddr = MacAddr([2, 0, 0, 0, 0, 0xfe]); + +/// The gateway's ARP reply to `net`. +fn gateway_reply(net: &Net) -> Vec { + use toyos_net_wire::arp::{Arp, Operation}; + use toyos_net_wire::ethernet::FrameBuilder; + let arp = Arp { operation: Operation::Reply, sender_mac: GATEWAY_MAC, sender_ip: LAN_GATEWAY, target_mac: net.mac, target_ip: net.addr }; + let mut frame = vec![0; 64]; + let len = FrameBuilder { destination: net.mac, source: mac(GATEWAY_MAC) }.emit(&arp, &mut frame).unwrap().len(); + frame.truncate(len); + frame +} + +/// A stale entry: resolved, then left past any reachable time. +fn stale(ip: &mut Ip, iface: IfIndex, net: &Net, now: Instant) -> Instant { + ip.resolve(now, iface, LAN_GATEWAY, net.addr); + ip.transmit(now, usize::MAX, |_, _| {}); + ip.receive(now, iface, &gateway_reply(net)); + let later = now.after(std::time::Duration::from_secs(120)); + while let Some(deadline) = ip.next_deadline().filter(|d| *d <= later) { + ip.fire(deadline); + ip.transmit(deadline, usize::MAX, |_, _| {}); + } + later +} + +#[test] +fn a_flows_advice_lands_on_its_own_links_gateway_where_two_links_share_one() { + let (mut ip, ifaces, now) = machine(&[(&LAN_WIRED, WIRED), (&LAN_WIRELESS, WIRELESS)]); + let now = stale(&mut ip, ifaces[0], &LAN_WIRED, now); + let now = stale(&mut ip, ifaces[1], &LAN_WIRELESS, now); + let state = |ip: &Ip, iface| ip.neighbour(iface, LAN_GATEWAY).map(|n| matches!(n, toyos_net_ip::Nud::Reachable(_))); + assert_eq!((state(&ip, ifaces[0]), state(&ip, ifaces[1])), (Some(false), Some(false)), "the premise: both stale"); + // A flow on the wireless link, from its address, confirms its next hop. + ip.advise(now, OFF_LINK, Source::Bound(LAN_WIRELESS.addr), toyos_net_ip::Advice::Confirmed); + assert_eq!((state(&ip, ifaces[0]), state(&ip, ifaces[1])), (Some(false), Some(true)), "the wireless entry, not the wired one"); +} + +/// An ARP request from host `n` on `net`'s /24 for `net`'s address. +fn asks_for(net: &Net, n: u8) -> Vec { + use toyos_net_wire::arp::{Arp, Operation}; + use toyos_net_wire::ethernet::FrameBuilder; + let [a, b, c, _] = net.addr.octets(); + let host = MacAddr([2, 9, 0, 0, 0, n]); + let arp = Arp { operation: Operation::Request, sender_mac: host, sender_ip: Ipv4Addr::new(a, b, c, 100u8.wrapping_add(n)), target_mac: MacAddr::ZERO, target_ip: net.addr }; + let mut frame = vec![0; 64]; + let len = FrameBuilder { destination: MacAddr::BROADCAST, source: mac(host) }.emit(&arp, &mut frame).unwrap().len(); + frame.truncate(len); + frame +} + +#[test] +fn a_link_that_takes_nothing_holds_back_none_of_another_links_own_frames() { + let (mut ip, ifaces, now) = machine(&[(&NET_1, WIRED), (&NET_2, WIRELESS)]); + let (stuck, other) = (ifaces[0], ifaces[1]); + // The wired link's share of the queue overfilled with its replies, none of which leave. + let full_before = ip.counters().get(Counter::IpControlQueueFull); + for n in 0..(toyos_net_ip::limits::CONTROL_QUEUE as u8 + 8) { + ip.receive(now, stuck, &asks_for(&NET_1, n)); + } + assert_eq!(ip.counters().get(Counter::IpControlQueueFull), full_before + 8, "the stuck link's own overflow is its own"); + // The other link's request is queued and leaves, past the stuck link's share. + ip.resolve(now, other, NET_2.gateway, NET_2.addr); + let mut out = Vec::new(); + ip.transmit(now, Only(other, usize::MAX), |iface, _| out.push(iface)); + assert_eq!(out, [other], "its request"); + // A second address on the other link: its probes leave and conflict detection ends. + let second = Ipv4Addr::new(198, 51, 100, 11); + ip.add_address(now, other, second, 24).unwrap(); + let until = now.after(std::time::Duration::from_secs(10)); + let mut probes = 0; + while let Some(deadline) = ip.next_deadline().filter(|d| *d <= until) { + ip.fire(deadline); + ip.transmit(deadline, Only(other, usize::MAX), |iface, _| { + assert_eq!(iface, other); + probes += 1; + }); + } + assert!(probes >= 3, "the probes and announcements left: {probes}"); + assert_eq!(ip.address(other, second), Some(toyos_net_ip::AddrState::Assigned)); +} diff --git a/system/netstack/node/src/lease.rs b/system/netstack/node/src/lease.rs index cfe12b47ffe..0da52207877 100644 --- a/system/netstack/node/src/lease.rs +++ b/system/netstack/node/src/lease.rs @@ -9,8 +9,8 @@ //! - a lease is held only against a [`Verified`], which only [`Stack::report`] makes, from [ip]'s //! report for the link's address under probe, so no address is in use before conflict //! detection ends; -//! - the resolvers lookups ask are one link's held lease's: of the links holding one, the first -//! up, by rank and then arrival ([`Stack::asked`]). The answers they gave go with every write of any +//! - the resolvers lookups ask are one link's held lease's: of the links up and holding one, the +//! first by rank and then arrival, and with none up the link asked before ([`Stack::asked`]). The answers they gave go with every write of any //! link's state, every link that goes down and every change of the link asked: after any of //! them the next lookup may be asking another network, so none is answered from this one's. //! @@ -70,6 +70,9 @@ pub(crate) struct Stack { socket: SocketId, /// In the order they were added. links: Vec, + /// The link lookups ask ([`Stack::asked`]), kept as each write of a link's state or link + /// leaves it. + asked: Option, /// What the asked link's resolvers answered: emptied with every write of a link's state, /// every link that goes down and every change of the link asked. answers: Cache, @@ -97,7 +100,7 @@ impl Stack { pub(crate) fn new(now: Instant, config: Config) -> Result { let mut shard = Shard::new(now, config)?; let Ok(socket) = shard.acquisition() else { unreachable!("a new shard has no socket on port 68") }; - Ok(Self { shard, socket, links: Vec::new(), answers: Cache::default(), inbox: VecDeque::new() }) + Ok(Self { shard, socket, links: Vec::new(), asked: None, answers: Cache::default(), inbox: VecDeque::new() }) } /// A link's interface, down and unaddressed. @@ -136,12 +139,23 @@ impl Stack { } } - /// The link lookups ask, of those holding a lease: one that is up before one that is not, - /// and then the first by rank and by when it was added. A link down keeps its lease, so with - /// none up the lookups keep asking the one they asked. + /// The link lookups ask: of the links up and holding a lease, the first by rank and then by + /// when it was added. With none such, the link asked before while it still holds its lease, + /// since a link down keeps it; and failing that the first by rank holding one. pub(crate) fn asked(&self) -> Option { + self.asked + } + + /// [`Self::asked`] chosen again after a link's state or link changed; `true` when it moved. + fn choose_asked(&mut self) -> bool { let held = |link: &&Link| matches!(link.state, State::Held(_)); - self.links.iter().filter(held).min_by_key(|link| (!link.up, link.rank)).map(|link| link.iface) + let up = self.links.iter().filter(held).filter(|link| link.up).min_by_key(|link| link.rank); + let kept = self.asked.and_then(|asked| self.links.iter().filter(held).find(|link| link.iface == asked)); + let any = self.links.iter().filter(held).min_by_key(|link| link.rank); + let asked = up.or(kept).or(any).map(|link| link.iface); + let moved = asked != self.asked; + self.asked = asked; + moved } /// The asked link's lease, whose resolvers lookups ask. @@ -160,6 +174,7 @@ impl Stack { /// Writes `iface`'s state; the answers go with it. fn set(&mut self, iface: IfIndex, state: State) { self.link_mut(iface).state = state; + self.choose_asked(); self.answers.empty(); } @@ -185,10 +200,9 @@ impl Stack { /// that the lease's verification never hears from. Up or down, a change of the link asked /// empties them too. pub(crate) fn link(&mut self, now: Instant, iface: IfIndex, up: bool) { - let asked = self.asked(); agreed(if up { self.shard.link_up(now, iface) } else { self.shard.link_down(now, iface) }); self.link_mut(iface).up = up; - if !up || self.asked() != asked { + if self.choose_asked() || !up { self.answers.empty(); } } diff --git a/system/netstack/node/src/lib.rs b/system/netstack/node/src/lib.rs index ce7a7c464de..7ca9ccc85fa 100644 --- a/system/netstack/node/src/lib.rs +++ b/system/netstack/node/src/lib.rs @@ -66,6 +66,7 @@ pub use resolve::{Ended, LookupId, NotStarted, Resolved, Resolving}; toyos_net_wire::counters! { DhcpUnsent = "node.dhcp-unsent"; + DhcpOtherLink = "node.dhcp-other-link"; AddressRefused = "node.address-refused"; RouterRefused = "node.router-refused"; NameUnsent = "node.name-unsent"; @@ -84,20 +85,28 @@ pub enum Event { Name { link: IfIndex, event: toyos_mdns::Event }, } +/// One link's DHCP client. +struct LinkClient { + link: IfIndex, + mac: IndividualMac, + client: Client, + /// Its refusals pass through it as the stack's pass through the shard's: one line a rule in + /// any 10 s, carrying how many were not. + log: toyos_dhcp::RefusalLog, +} + pub struct Node { stack: Stack, /// The name each link's client asks its network to record. host_name: Option, - /// Each link's DHCP client, in the order the links were added. - clients: Vec<(IfIndex, Client)>, + /// Each link's DHCP client, its link's address and the log its refusals pass through, in + /// the order the links were added. + clients: Vec, /// The responders for the machine's name, once [`Self::answer_as`] started them. name: Option, resolver: resolve::Resolver, counters: Counters, events: Vec, - /// The clients' refusals pass through it as the stack's pass through the shard's: one line a - /// rule in any 10 s, carrying how many were not. - dhcp_log: toyos_dhcp::RefusalLog, /// Room for the largest message the client accepts. datagram: Vec, streams: streams::Streams, @@ -121,7 +130,6 @@ impl Node { resolver: resolve::Resolver::new(), counters: Counters::default(), events: Vec::new(), - dhcp_log: toyos_dhcp::RefusalLog::default(), datagram: vec![0; usize::from(toyos_dhcp::limits::MAX_MESSAGE)], streams: streams::Streams::default(), listeners: listeners::Listeners::default(), @@ -136,7 +144,7 @@ impl Node { pub fn add_link(&mut self, now: Instant, mac: IndividualMac, rank: Rank, draw: impl FnMut() -> u32) -> IfIndex { let iface = self.stack.add_link(now, mac, rank); let (client, _unsent) = Client::start(now, mac, self.host_name.clone(), draw); - self.clients.push((iface, client)); + self.clients.push(LinkClient { link: iface, mac, client, log: toyos_dhcp::RefusalLog::default() }); if let Some(name) = &mut self.name { name.add_link(now, &mut self.stack, iface); } @@ -173,8 +181,8 @@ impl Node { } fn client(&mut self, link: IfIndex) -> &mut Client { - match self.clients.iter_mut().find(|(at, _)| *at == link) { - Some((_, client)) => client, + match self.clients.iter_mut().find(|c| c.link == link) { + Some(c) => &mut c.client, None => unreachable!("{link:?} is no link of the node's"), } } @@ -244,7 +252,7 @@ impl Node { pub fn next_deadline(&self) -> Option { let name = self.name.as_ref().and_then(name::Name::next_deadline); - let clients = self.clients.iter().filter_map(|(_, client)| client.next_deadline()); + let clients = self.clients.iter().filter_map(|c| c.client.next_deadline()); self.stack.next_deadline().into_iter().chain(clients).chain(name).chain(self.resolver.next_deadline()).chain(self.streams.next_deadline()).min() } @@ -253,9 +261,9 @@ impl Node { self.stack.fire(now); self.settle(now, &mut draw); for at in 0..self.clients.len() { - let Some((link, client)) = self.clients.get_mut(at) else { continue }; - let link = *link; - let out = client.timer(now, &mut draw); + let Some(c) = self.clients.get_mut(at) else { continue }; + let link = c.link; + let out = c.client.timer(now, &mut draw); self.carry_out(now, link, out, None, &mut draw); } self.settle(now, &mut draw); @@ -276,8 +284,16 @@ impl Node { Report::NotVerified => (link, client.not_verified(now, &mut *draw), None), } } else if let Some((payload, from, link)) = self.stack.recv(&mut self.datagram) { - let Some((_, client)) = self.clients.iter_mut().find(|(at, _)| *at == link) else { unreachable!("[ip] delivered from no link of the node's") }; - (link, client.receive(now, payload, from, &mut *draw), None) + // On a LAN two of the links share, a reply for one link's client reaches the + // other's too: it is that link's, and this one's client never hears it. + let other = toyos_dhcp::addressed_to(payload) + .is_some_and(|to| self.clients.iter().any(|c| c.link != link && c.mac.get() == to)); + if other { + self.counters.add(Counter::DhcpOtherLink, 1); + continue; + } + let Some(c) = self.clients.iter_mut().find(|c| c.link == link) else { unreachable!("[ip] delivered from no link of the node's") }; + (link, c.client.receive(now, payload, from, &mut *draw), None) } else { break; }; @@ -292,10 +308,10 @@ impl Node { fn log(&mut self, now: Instant) { let events = &mut self.events; self.stack.refusals(|refusal, suppressed| events.push(Event::Stack { refusal, suppressed })); - for (link, client) in &mut self.clients { - for refusal in client.drain_refusals() { - if let Some(suppressed) = self.dhcp_log.admit(now, refusal.rule) { - events.push(Event::Dhcp { link: *link, refusal, suppressed }); + for c in &mut self.clients { + for refusal in c.client.drain_refusals() { + if let Some(suppressed) = c.log.admit(now, refusal.rule) { + events.push(Event::Dhcp { link: c.link, refusal, suppressed }); } } } diff --git a/system/netstack/node/src/name.rs b/system/netstack/node/src/name.rs index 7f2e33c92e8..806ec74fa44 100644 --- a/system/netstack/node/src/name.rs +++ b/system/netstack/node/src/name.rs @@ -6,6 +6,10 @@ //! - **One name, a responder per link.** A message is its arrival link's responder's alone, and //! what a responder sends leaves by its link alone. A name lost on one link is that link's: the //! responder there probes for the same name again later (§8.1), and the others keep it. +//! - **A message from one of this machine's own addresses never reaches a responder**: [ip] +//! refuses a datagram whose source any of its interfaces holds (`ip.own-source`), so on one LAN +//! two links share, as a wired and a wireless link bridged (§14), this machine hears its own +//! messages from neither and never conflicts with itself. //! - The record is the link's held lease's on a link that is up: its address and prefix are what the //! responder is told, and it is told it has no link from the moment the lease goes or the link //! does. Nothing is probed for, announced or answered for an address still under probe, or on a diff --git a/system/netstack/node/tests/links.rs b/system/netstack/node/tests/links.rs index 3a38864f65d..7485fbb0465 100644 --- a/system/netstack/node/tests/links.rs +++ b/system/netstack/node/tests/links.rs @@ -32,6 +32,9 @@ struct Net { } const WIRED_NET: Net = Net { mac: [2, 0, 0, 0, 0, 0x01], addr: Ipv4Addr::new(192, 0, 2, 10), router: Ipv4Addr::new(192, 0, 2, 1), dns: Ipv4Addr::new(192, 0, 2, 53) }; +/// A wired and a wireless link on one LAN: one router and server, one resolver. +const LAN_WIRED: Net = Net { mac: [2, 0, 0, 0, 0, 0x01], addr: Ipv4Addr::new(192, 168, 1, 10), router: Ipv4Addr::new(192, 168, 1, 1), dns: Ipv4Addr::new(192, 168, 1, 53) }; +const LAN_WIRELESS: Net = Net { mac: [2, 0, 0, 0, 0, 0x02], addr: Ipv4Addr::new(192, 168, 1, 20), router: Ipv4Addr::new(192, 168, 1, 1), dns: Ipv4Addr::new(192, 168, 1, 53) }; const WIRELESS_NET: Net = Net { mac: [2, 0, 0, 0, 0, 0x02], addr: Ipv4Addr::new(198, 51, 100, 10), router: Ipv4Addr::new(198, 51, 100, 1), dns: Ipv4Addr::new(198, 51, 100, 53) }; @@ -53,7 +56,7 @@ fn read(frame: &[u8]) -> Out { let v4 = |b: &[u8]| Ipv4Addr::new(b[0], b[1], b[2], b[3]); Out::ArpRequest { sender: v4(arp.sender_protocol_addr()), target: v4(arp.target_protocol_addr()) } } - (Some(NetSlice::Ipv4(ip)), Some(TransportSlice::Udp(udp))) if udp.destination_port() == 67 => { + (Some(NetSlice::Ipv4(_)), Some(TransportSlice::Udp(udp))) if udp.destination_port() == 67 => { let message = udp.payload(); let xid = u32::from_be_bytes(message[4..8].try_into().unwrap()); let mut at = 240; @@ -64,7 +67,6 @@ fn read(frame: &[u8]) -> Out { } at += 2 + len; }; - let _ = ip; Out::Dhcp { kind, xid } } (Some(NetSlice::Ipv4(ip)), Some(TransportSlice::Udp(udp))) => { @@ -93,8 +95,8 @@ fn arp_reply(net: &Net, addr: Ipv4Addr) -> Vec { } /// `net`'s server's `kind` message for transaction `xid`, offering or acknowledging its address -/// for an hour, with its router and resolver. -fn dhcp(net: &Net, kind: u8, xid: u32) -> Vec { +/// for an hour, with its router and resolver; to the client's own addresses, or broadcast. +fn dhcp(net: &Net, kind: u8, xid: u32, broadcast: bool) -> Vec { let mut m = vec![2, 1, 6, 0]; m.extend_from_slice(&xid.to_be_bytes()); m.extend_from_slice(&[0; 8]); @@ -117,7 +119,10 @@ fn dhcp(net: &Net, kind: u8, xid: u32) -> Vec { m.extend_from_slice(&value); } m.push(255); - udp_frame(net.mac, ROUTER_MAC, (net.router, 67), (net.addr, 68), &m) + match broadcast { + false => udp_frame(net.mac, ROUTER_MAC, (net.router, 67), (net.addr, 68), &m), + true => udp_frame([0xff; 6], ROUTER_MAC, (net.router, 67), (Ipv4Addr::BROADCAST, 68), &m), + } } struct Two { @@ -128,6 +133,11 @@ struct Two { draws: u32, /// Every frame the node sent, with the link it left by. sent: Vec<(IfIndex, Out)>, + /// The two links are bridged onto one LAN: a group or broadcast frame sent on one is heard + /// on the other, and a server's reply reaches both. + bridged: bool, + wired_net: &'static Net, + wireless_net: &'static Net, } fn counter(draws: &mut u32) -> impl FnMut() -> u32 + '_ { @@ -150,11 +160,20 @@ impl Two { node.set_places(8); let wireless = node.add_link(now, IndividualMac::new(MacAddr(WIRELESS_NET.mac)).unwrap(), WIRELESS, counter(&mut draws)); let wired = node.add_link(now, IndividualMac::new(MacAddr(WIRED_NET.mac)).unwrap(), WIRED, counter(&mut draws)); - Self { node, wired, wireless, now, draws, sent: Vec::new() } + Self { node, wired, wireless, now, draws, sent: Vec::new(), bridged: false, wired_net: &WIRED_NET, wireless_net: &WIRELESS_NET } + } + + /// Both links on one LAN behind one server. + fn bridged() -> Self { + Self { bridged: true, wired_net: &LAN_WIRED, wireless_net: &LAN_WIRELESS, ..Self::new() } } fn net(&self, link: IfIndex) -> &'static Net { - if link == self.wired { &WIRED_NET } else { &WIRELESS_NET } + if link == self.wired { self.wired_net } else { self.wireless_net } + } + + fn other(&self, link: IfIndex) -> IfIndex { + if link == self.wired { self.wireless } else { self.wired } } fn deliver(&mut self, link: IfIndex, frame: &[u8]) { @@ -168,10 +187,28 @@ impl Two { fn pump(&mut self) { loop { let mut out = Vec::new(); - self.node.transmit(self.now, usize::MAX, |link, frame| out.push((link, read(frame))), counter(&mut self.draws)); + let mut heard = Vec::new(); + let bridged = self.bridged; + self.node.transmit( + self.now, + usize::MAX, + |link, frame| { + out.push((link, read(frame))); + // A group or broadcast frame's destination has its low bit of the first byte set. + if bridged && frame[0] & 1 == 1 { + heard.push((link, frame.to_vec())); + } + }, + counter(&mut self.draws), + ); if out.is_empty() { return; } + for (link, frame) in heard { + let other = self.other(link); + let mut frames = Some(frame.as_slice()); + self.node.receive(self.now, other, 256, |sink| frames.take().map(&mut *sink).is_some(), counter(&mut self.draws)); + } let replies: Vec<(IfIndex, Vec)> = out .iter() .filter_map(|(link, seen)| { @@ -222,23 +259,37 @@ impl Two { .expect("a DHCP message of that type on the link") } + /// A server's reply on `link`, which on a bridged LAN reaches the other link too. + fn reply(&mut self, link: IfIndex, frame: &[u8]) { + if self.bridged { + let other = self.other(link); + self.deliver(other, frame); + } + self.deliver(link, frame); + } + /// `link` up, and its server's offer and acknowledgement through to a lease held. fn lease(&mut self, link: IfIndex) { self.link(link, true); let net = self.net(link); let xid = self.last_dhcp(link, 1); - self.deliver(link, &dhcp(net, 2, xid)); + // A server on a LAN two links share broadcasts its replies, so each link hears both. + let broadcast = self.bridged; + self.reply(link, &dhcp(net, 2, xid, broadcast)); assert_eq!(self.last_dhcp(link, 3), xid, "the REQUEST"); - self.deliver(link, &dhcp(net, 5, xid)); + self.reply(link, &dhcp(net, 5, xid, broadcast)); assert!(self.run_until(Duration::from_secs(10), |two| two.node.lease(link).is_some()), "the lease is held"); } fn leased() -> Self { - let mut two = Self::new(); - let (wired, wireless) = (two.wired, two.wireless); - two.lease(wireless); - two.lease(wired); - two + Self::new().leasing() + } + + fn leasing(mut self) -> Self { + let (wired, wireless) = (self.wired, self.wireless); + self.lease(wireless); + self.lease(wired); + self } /// The DNS queries that left from `from`, each with its link and resolver. @@ -291,13 +342,42 @@ fn lookups_ask_the_wired_links_resolver_fail_over_and_return() { assert_eq!(two.queries(from), [(wired, WIRED_NET.dns)]); } +/// Whichever link goes down last, lookups keep asking the link they asked while it was up. #[test] fn with_no_link_up_lookups_ask_the_lease_they_asked() { let mut two = Two::leased(); - let wired = two.wired; - two.link(two.wireless, false); + let (wired, wireless) = (two.wired, two.wireless); + two.link(wireless, false); + two.link(wired, false); + assert_eq!(two.node.asked(), Some(wired), "the wireless link down first, then the wired one it asked"); + let mut two = Two::leased(); two.link(wired, false); - assert_eq!(two.node.asked(), Some(wired)); + assert_eq!(two.node.asked(), Some(wireless), "failed over"); + two.link(wireless, false); + assert_eq!(two.node.asked(), Some(wireless), "the wired link down first: the wireless one it then asked"); +} + +/// RFC 6762 §14's laptop: a wired and a wireless link bridged onto one LAN. Each link hears +/// the other's group frames and the server's replies to both, and the machine claims its name +/// on both and refuses no server's reply. +#[test] +fn on_one_lan_by_two_links_the_name_is_claimed_on_both_and_no_reply_is_refused() { + let mut two = Two::bridged().leasing(); + let (wired, wireless) = (two.wired, two.wireless); + assert_eq!(two.node.lease(wired).map(|l| l.address), Some(LAN_WIRED.addr)); + assert_eq!(two.node.lease(wireless).map(|l| l.address), Some(LAN_WIRELESS.addr)); + two.node.answer_as(two.now, Host::new("toyos").unwrap(), counter(&mut two.draws)).unwrap(); + two.pump(); + let mut events: Vec = two.node.drain_events().collect(); + let claimed = |events: &[Event], link| events.contains(&Event::Name { link, event: toyos_mdns::Event::Claimed }); + assert!(two.run_until(Duration::from_secs(10), |two| { + events.extend(two.node.drain_events()); + claimed(&events, wired) && claimed(&events, wireless) + }), "claimed on both links: {events:?}"); + assert!(!events.iter().any(|e| matches!(e, Event::Name { event: toyos_mdns::Event::Lost, .. })), "{events:?}"); + assert!(!events.iter().any(|e| matches!(e, Event::Dhcp { .. })), "no server's reply refused: {events:?}"); + assert!(two.node.shard().ip().counters().get(toyos_net_ip::Counter::IpOwnSource) > 0, "the premise: each link heard the other's messages, which [ip] refused"); + assert!(two.node.counters().get(toyos_net_node::Counter::DhcpOtherLink) > 0, "the premise: each link heard the other's replies"); } const NAME: &[u8] = b"\x05toyos\x05local\x00"; diff --git a/system/netstack/shard/Cargo.toml b/system/netstack/shard/Cargo.toml index 72f53410b32..75a370c3960 100644 --- a/system/netstack/shard/Cargo.toml +++ b/system/netstack/shard/Cargo.toml @@ -3,7 +3,7 @@ name = "toyos-net-shard" version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" -description = "One shard of ToyOS's network stack, pure: toyos-net-ip, toyos-net-tcp and toyos-net-udp over one interface, frames in and frames pulled out with the device's credit." +description = "One shard of ToyOS's network stack, pure: toyos-net-ip, toyos-net-tcp and toyos-net-udp over every interface the caller adds, frames in by interface and frames pulled out with each link's credit." publish = false [dependencies] diff --git a/system/netstack/shard/src/lib.rs b/system/netstack/shard/src/lib.rs index 39dc48fd3bf..9a7927d89e7 100644 --- a/system/netstack/shard/src/lib.rs +++ b/system/netstack/shard/src/lib.rs @@ -129,7 +129,7 @@ const TCP_MTU: u16 = { /// on the link its route leaves by. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] enum Wait { - Hop(Ipv4Addr), + Hop(IfIndex, Ipv4Addr), Room, Link(IfIndex), } @@ -285,7 +285,7 @@ impl Shard { for iface in room { for hop in self.waiting.remove(&Wait::Link(iface)).into_iter().flatten() { self.tcp.wake(hop); - self.udp.wake(hop); + self.udp.wake(iface, hop); } } self.settle(now); @@ -388,10 +388,10 @@ impl Shard { }); Offer::Taken } - Ok(Sent::Pending(next_hop)) => Offer::Waits(next_hop), + Ok(Sent::Pending { iface, next_hop }) => Offer::Waits { iface, hop: next_hop }, Ok(Sent::NoRoom { iface, hop }) => { waiting.entry(Wait::Link(iface)).or_default().insert(hop); - Offer::Waits(hop) + Offer::Waits { iface, hop } } // Counted by [ip], which tells the datagram's flow. Err(_) => Offer::Taken, @@ -425,8 +425,8 @@ impl Shard { events.push(Event::Refused { refusal: Refusal::Tcp(r), suppressed }); } } - toyos_net_tcp::Event::Reachable(peer) => ip.advise(now, peer, Advice::Confirmed), - toyos_net_tcp::Event::Reverify(peer) => ip.advise(now, peer, Advice::Reverify), + toyos_net_tcp::Event::Reachable { local, remote } => ip.advise(now, remote, Source::Bound(local), Advice::Confirmed), + toyos_net_tcp::Event::Reverify { local, remote } => ip.advise(now, remote, Source::Bound(local), Advice::Reverify), } } for event in ip.drain_events() { @@ -437,16 +437,16 @@ impl Shard { } } toyos_net_ip::Event::Unreachable(flow) => udp.unreachable(&flow), - toyos_net_ip::Event::Resolved { next_hop, .. } => { - wake(tcp, waiting, Wait::Hop(next_hop)); - udp.wake(next_hop); + toyos_net_ip::Event::Resolved { iface, next_hop } => { + wake(tcp, waiting, Wait::Hop(iface, next_hop)); + udp.wake(iface, next_hop); } - toyos_net_ip::Event::Failed { next_hop, .. } => { - wake(tcp, waiting, Wait::Hop(next_hop)); - udp.fail(next_hop); + toyos_net_ip::Event::Failed { iface, next_hop } => { + wake(tcp, waiting, Wait::Hop(iface, next_hop)); + udp.fail(iface, next_hop); } // No datagram waits on a failed next hop: the failure dropped them. - toyos_net_ip::Event::Cleared { next_hop, .. } => wake(tcp, waiting, Wait::Hop(next_hop)), + toyos_net_ip::Event::Cleared { iface, next_hop } => wake(tcp, waiting, Wait::Hop(iface, next_hop)), toyos_net_ip::Event::Room { .. } => wake(tcp, waiting, Wait::Room), toyos_net_ip::Event::Verified { iface, addr } => events.push(Event::Verified { iface, addr }), toyos_net_ip::Event::Conflict { iface, addr, mac } => events.push(Event::Conflict { iface, addr, mac }), @@ -759,12 +759,12 @@ fn hop(ip: &mut Ip, now: Instant, credit: &impl Credit, tuple: &Tuple, waiting: let (answer, wait) = match ip.neighbour(iface, next_hop) { Some(entry) => match entry.mac() { Some(mac) => (Hop::Ready(Via { iface, source, next_hop, mac }), None), - None if matches!(entry, Nud::Failed) => (Hop::Unreachable, Some(Wait::Hop(next_hop))), - None => (Hop::Pending, Some(Wait::Hop(next_hop))), + None if matches!(entry, Nud::Failed) => (Hop::Unreachable, Some(Wait::Hop(iface, next_hop))), + None => (Hop::Pending, Some(Wait::Hop(iface, next_hop))), }, None => match ip.resolve(now, iface, next_hop, route.source) { Resolution::Resolved(mac) => (Hop::Ready(Via { iface, source, next_hop, mac }), None), - Resolution::Pending => (Hop::Pending, Some(Wait::Hop(next_hop))), + Resolution::Pending => (Hop::Pending, Some(Wait::Hop(iface, next_hop))), Resolution::Failed => (Hop::Unreachable, Some(Wait::Room)), }, }; diff --git a/system/netstack/shard/tcp/src/conn.rs b/system/netstack/shard/tcp/src/conn.rs index cd76d9b2705..f73b25bc400 100644 --- a/system/netstack/shard/tcp/src/conn.rs +++ b/system/netstack/shard/tcp/src/conn.rs @@ -779,7 +779,7 @@ impl Sync { self.acked_since = Some(now); self.stalled = 0; self.orphan_progress(now); - ctx.log.event(Event::Reachable(ctx.tuple.remote.addr)); + ctx.log.event(Event::Reachable { local: ctx.tuple.local.addr, remote: ctx.tuple.remote.addr }); let smss = self.smss(); match self.recovery { Recovery::Fast { .. } if ack.sub(1).at_or_after(self.recover) => { @@ -1225,7 +1225,7 @@ impl Sync { self.stalled = self.stalled.saturating_add(1); if self.stalled == 3 { self.delivery_problem = true; - ctx.log.event(Event::Reverify(ctx.tuple.remote.addr)); + ctx.log.event(Event::Reverify { local: ctx.tuple.local.addr, remote: ctx.tuple.remote.addr }); } } diff --git a/system/netstack/shard/tcp/src/counters.rs b/system/netstack/shard/tcp/src/counters.rs index 5211c3faf39..19865bb0566 100644 --- a/system/netstack/shard/tcp/src/counters.rs +++ b/system/netstack/shard/tcp/src/counters.rs @@ -116,18 +116,18 @@ impl Log { /// Past [`limits::EVENTS`] undrained, an event is refused and counted: a peer never grows /// the list. A `Reachable` is not repeated while it is the newest advice pending for its - /// address: one peer's ACKs never crowd out another's refusals, and [ip] still reads each - /// address's advice in order. + /// pair of addresses: one peer's ACKs never crowd out another's refusals, and [ip] still + /// reads each pair's advice in order. pub fn event(&mut self, event: Event) { match event { - Event::Reachable(addr) if self.newest_advice(addr) == Some(&event) => {} + Event::Reachable { local, remote } if self.newest_advice(local, remote) == Some(&event) => {} _ if self.events.len() >= limits::EVENTS => self.count(Counter::EventOverflow), _ => self.events.push(event), } } - fn newest_advice(&self, addr: Ipv4Addr) -> Option<&Event> { - self.events.iter().rev().find(|e| matches!(e, Event::Reachable(a) | Event::Reverify(a) if *a == addr)) + fn newest_advice(&self, local: Ipv4Addr, remote: Ipv4Addr) -> Option<&Event> { + self.events.iter().rev().find(|e| matches!(e, Event::Reachable { local: l, remote: r } | Event::Reverify { local: l, remote: r } if *l == local && *r == remote)) } pub fn drain(&mut self) -> alloc::vec::Drain<'_, Event> { diff --git a/system/netstack/shard/tcp/src/lib.rs b/system/netstack/shard/tcp/src/lib.rs index cb131323458..8662de061d1 100644 --- a/system/netstack/shard/tcp/src/lib.rs +++ b/system/netstack/shard/tcp/src/lib.rs @@ -316,10 +316,11 @@ impl Hop { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Event { Refused(Refusal), - /// An ACK advanced SND.UNA: the next hop to this address answers (RFC 1122 §3.3.1.4). - Reachable(Ipv4Addr), - /// Three expiries without progress: re-verify the next hop to this address. - Reverify(Ipv4Addr), + /// An ACK advanced SND.UNA: the next hop to `remote` from `local` answers (RFC 1122 + /// §3.3.1.4). The local address names the link: one remote may be reached by several. + Reachable { local: Ipv4Addr, remote: Ipv4Addr }, + /// Three expiries without progress: re-verify the next hop to `remote` from `local`. + Reverify { local: Ipv4Addr, remote: Ipv4Addr }, } // Each compile_fail block sits beside one that compiles, so a typo cannot pass it. diff --git a/system/netstack/shard/tcp/src/open.rs b/system/netstack/shard/tcp/src/open.rs index 7322f850390..644a10e2cc4 100644 --- a/system/netstack/shard/tcp/src/open.rs +++ b/system/netstack/shard/tcp/src/open.rs @@ -145,7 +145,7 @@ impl Retransmit { self.owed = true; self.stalled = self.stalled.saturating_add(1); if self.stalled == 3 { - ctx.log.event(crate::Event::Reverify(ctx.tuple.remote.addr)); + ctx.log.event(crate::Event::Reverify { local: ctx.tuple.local.addr, remote: ctx.tuple.remote.addr }); } false } diff --git a/system/netstack/shard/tcp/tests/modern.rs b/system/netstack/shard/tcp/tests/modern.rs index 1ac1dc56680..6e879679b8e 100644 --- a/system/netstack/shard/tcp/tests/modern.rs +++ b/system/netstack/shard/tcp/tests/modern.rs @@ -88,7 +88,7 @@ fn s_mod_005_a_bulk_peer_leaves_room_for_refusals() { h.arrive(seg(7000).syn().rst().from(Ipv4Addr::new(192, 0, 2, 3), 1234).to(A, 81)); h.transmit(); assert_eq!(h.info().snd_una.get(), 1001 + limits::EVENTS as u32 + 1); - assert_eq!(h.events.iter().filter(|e| **e == Event::Reachable(B)).count(), 1); + assert_eq!(h.events.iter().filter(|e| **e == Event::Reachable { local: A, remote: B }).count(), 1); assert_eq!(h.refusals(Counter::SynRst).len(), 1); assert_eq!(h.count(Counter::EventOverflow), 0); } diff --git a/system/netstack/shard/tcp/tests/timers.rs b/system/netstack/shard/tcp/tests/timers.rs index 133a5f07153..76688ff0a88 100644 --- a/system/netstack/shard/tcp/tests/timers.rs +++ b/system/netstack/shard/tcp/tests/timers.rs @@ -92,10 +92,10 @@ fn s_gu_001_three_expiries_raise_advice() { h.send(0, 100); h.at(1399); assert!(!h.status().delivery_problem); - assert!(!h.events.contains(&Event::Reverify(B))); + assert!(!h.events.contains(&Event::Reverify { local: A, remote: B })); h.at(1400); assert!(h.status().delivery_problem); - assert_eq!(h.events.iter().filter(|e| **e == Event::Reverify(B)).count(), 1); + assert_eq!(h.events.iter().filter(|e| **e == Event::Reverify { local: A, remote: B }).count(), 1); } #[test] @@ -163,7 +163,7 @@ fn s_gu_006_progress_confirms_the_next_hop() { h.input(10, seg(5001).ack(2461)); h.input(11, seg(5001).ack(2461)); h.input(12, seg(5001).ack(3921)); - assert_eq!(h.events.iter().filter(|e| **e == Event::Reachable(B)).count(), 2); + assert_eq!(h.events.iter().filter(|e| **e == Event::Reachable { local: A, remote: B }).count(), 2); } /// [ip] reads the advice in order: an advance after a pending re-verify confirms the next hop again. @@ -182,7 +182,7 @@ fn s_gu_006_a_confirmation_after_a_reverify_is_kept() { } h.arrive(seg(5001).ack(3921)); let events: Vec = h.tcp.drain_events().collect(); - assert_eq!(events, [Event::Reachable(B), Event::Reverify(B), Event::Reachable(B)]); + assert_eq!(events, [Event::Reachable { local: A, remote: B }, Event::Reverify { local: A, remote: B }, Event::Reachable { local: A, remote: B }]); } /// [ip] reads each address's own advice: B's repeated confirmation never crowds out C's, and a @@ -204,5 +204,5 @@ fn s_gu_006_b_advice_is_kept_per_address() { h.arrive(seg(5001).ack(1101).from(C, 80).to(A, 49153)); h.arrive(seg(5001).ack(1201)); let events: Vec = h.tcp.drain_events().collect(); - assert_eq!(events, [Event::Reachable(B), Event::Reachable(C)]); + assert_eq!(events, [Event::Reachable { local: A, remote: B }, Event::Reachable { local: A, remote: C }]); } diff --git a/system/netstack/shard/tests/links.rs b/system/netstack/shard/tests/links.rs index d9f472ee47e..6a3ba19e1c5 100644 --- a/system/netstack/shard/tests/links.rs +++ b/system/netstack/shard/tests/links.rs @@ -168,3 +168,167 @@ fn a_link_that_only_goes_down_keeps_its_address() { m.shard.link_down(m.now, m.wired).unwrap(); assert!(m.shard.ip().address(m.wired, WIRED).is_some()); } + +// ---- one LAN on two links: one gateway address on each ---- + +const LAN_GATEWAY: Ipv4Addr = Ipv4Addr::new(192, 168, 1, 1); +const LAN_WIRED: Ipv4Addr = Ipv4Addr::new(192, 168, 1, 10); +const LAN_WIRELESS: Ipv4Addr = Ipv4Addr::new(192, 168, 1, 20); + +/// Both links on one /24, each with the same gateway address, and a datagram socket bound to +/// each link's address. +fn one_lan() -> (Machine, toyos_net_udp::SocketId, toyos_net_udp::SocketId) { + let mut now = Instant::from_millis(3_600_000); + let mut shard = Shard::new(now, Config { receive_buffer: 65_535, send_buffer: 65_535, secrets: secrets() }).unwrap(); + let wired = shard.add_interface(now, IndividualMac::new(WIRED_MAC).unwrap(), Rank(0)); + let wireless = shard.add_interface(now, IndividualMac::new(WIRELESS_MAC).unwrap(), Rank(1)); + for (iface, addr) in [(wired, LAN_WIRED), (wireless, LAN_WIRELESS)] { + shard.link_up(now, iface).unwrap(); + shard.add_address(now, iface, addr, 24).unwrap(); + } + let mut verified = 0; + while verified < 2 { + shard.transmit(now, usize::MAX, |_, _| {}); + verified += shard.drain_events().filter(|e| matches!(e, Event::Verified { .. })).count(); + if verified < 2 { + now = shard.next_deadline().expect("conflict detection is still running"); + shard.fire(now); + } + } + for iface in [wired, wireless] { + shard.set_gateways(now, iface, &[LAN_GATEWAY]).unwrap(); + } + let from_wired = shard.bind(LAN_WIRED, Port::new(5000), || 0).unwrap(); + let from_wireless = shard.bind(LAN_WIRELESS, Port::new(5001), || 0).unwrap(); + (Machine { shard, wired, wireless, now }, from_wired, from_wireless) +} + +/// The UDP datagrams that left in one opportunity, by interface and source; ARP answered by +/// nobody. +fn datagrams(m: &mut Machine) -> Vec<(IfIndex, Ipv4Addr)> { + let mut out = Vec::new(); + m.shard.transmit(m.now, usize::MAX, |iface, bytes| { + let frame = Frame::parse(bytes).unwrap(); + if matches!(frame.ether_type(), EtherType::Ipv4) { + out.push((iface, Ipv4Packet::parse(frame.body()).unwrap().source())); + } + }); + out +} + +#[test] +fn a_gateway_that_fails_on_one_link_drops_no_datagram_waiting_on_the_other() { + let (mut m, from_wired, from_wireless) = one_lan(); + let (wired, wireless) = (m.wired, m.wireless); + m.shard.send_to(m.now, from_wired, FAR, 9, b"wired").unwrap(); + assert!(datagrams(&mut m).is_empty(), "the wired datagram waits for the gateway's link address"); + // The wireless link asks a retransmission later, so its question is still open when the + // wired link gives the gateway up; nobody answers the wired one, and the wireless one only + // after. + m.now = m.shard.next_deadline().expect("the wired link's resolution is running"); + m.shard.fire(m.now); + datagrams(&mut m); + m.shard.send_to(m.now, from_wireless, FAR, 9, b"wireless").unwrap(); + assert!(datagrams(&mut m).is_empty(), "the wireless datagram waits for the same address on its own link"); + let failed = m.shard.udp_counters().get(toyos_net_udp::Counter::TxUnreachable); + while m.shard.udp_counters().get(toyos_net_udp::Counter::TxUnreachable) == failed { + m.now = m.shard.next_deadline().expect("the wired link's resolution is still running"); + m.shard.fire(m.now); + datagrams(&mut m); + } + assert_eq!(m.shard.udp_counters().get(toyos_net_udp::Counter::TxUnreachable), failed + 1, "the wired datagram alone"); + m.shard.receive(m.now, wireless, &router_reply(LAN_GATEWAY, (WIRELESS_MAC, LAN_WIRELESS))); + assert_eq!(datagrams(&mut m), [(wireless, LAN_WIRELESS)], "the wireless datagram leaves"); + assert!(m.shard.ip().neighbour(wired, LAN_GATEWAY).is_some_and(|n| matches!(n, toyos_net_ip::Nud::Failed)), "the premise: the wired link gave it up"); +} + +/// The ones'-complement sum of `bytes` (RFC 1071). +fn checksum(bytes: &[u8]) -> u16 { + let mut sum: u32 = bytes.chunks(2).map(|c| u32::from(u16::from_be_bytes([c[0], *c.get(1).unwrap_or(&0)]))).sum(); + while sum >> 16 != 0 { + sum = (sum & 0xffff) + (sum >> 16); + } + !(sum as u16) +} + +/// A bare TCP segment from `from` to `to` in a frame from the gateway to `mac`. +fn segment(mac: MacAddr, from: (Ipv4Addr, u16), to: (Ipv4Addr, u16), seq: u32, ack: u32, flags: u8) -> Vec { + let mut tcp = Vec::new(); + tcp.extend_from_slice(&from.1.to_be_bytes()); + tcp.extend_from_slice(&to.1.to_be_bytes()); + tcp.extend_from_slice(&seq.to_be_bytes()); + tcp.extend_from_slice(&ack.to_be_bytes()); + tcp.extend_from_slice(&[0x50, flags, 0xff, 0xff, 0, 0, 0, 0]); + let pseudo = [&from.0.octets()[..], &to.0.octets(), &[0, 6, 0, 20]].concat(); + let check = checksum(&[pseudo, tcp.clone()].concat()); + tcp[16..18].copy_from_slice(&check.to_be_bytes()); + let mut ip = vec![0x45, 0, 0, 40, 0, 0, 0x40, 0, 64, 6, 0, 0]; + ip.extend_from_slice(&from.0.octets()); + ip.extend_from_slice(&to.0.octets()); + let check = checksum(&ip); + ip[10..12].copy_from_slice(&check.to_be_bytes()); + [&mac.0[..], &ROUTER_MAC.0, &[0x08, 0x00], &ip, &tcp].concat() +} + +fn gateway_state(m: &Machine, iface: IfIndex) -> Option<&'static str> { + m.shard.ip().neighbour(iface, LAN_GATEWAY).map(|n| match n { + toyos_net_ip::Nud::Reachable(_) => "reachable", + toyos_net_ip::Nud::Stale(_) => "stale", + toyos_net_ip::Nud::Delay(_) => "delay", + _ => "other", + }) +} + +/// Resolves the gateway on each link, a datagram from each link's socket asking, and leaves +/// both entries stale. +fn both_stale(m: &mut Machine, from_wired: toyos_net_udp::SocketId, from_wireless: toyos_net_udp::SocketId) { + let (wired, wireless) = (m.wired, m.wireless); + for (socket, iface, mine) in [(from_wired, wired, (WIRED_MAC, LAN_WIRED)), (from_wireless, wireless, (WIRELESS_MAC, LAN_WIRELESS))] { + m.shard.send_to(m.now, socket, FAR, 9, b"x").unwrap(); + datagrams(m); + m.shard.receive(m.now, iface, &router_reply(LAN_GATEWAY, mine)); + datagrams(m); + } + let later = m.now.after(std::time::Duration::from_secs(120)); + while let Some(at) = m.shard.next_deadline().filter(|at| *at <= later) { + m.now = at; + m.shard.fire(at); + datagrams(m); + } + m.now = later; + assert_eq!((gateway_state(m, wired), gateway_state(m, wireless)), (Some("stale"), Some("stale")), "the premise: both stale"); +} + +/// A connection's progress confirms the next hop of its own link: where the wired and the +/// wireless link share one gateway address, a flow on the wireless link refreshes the wireless +/// entry and leaves the wired one as it was. +#[test] +fn a_flows_progress_confirms_its_own_links_gateway_where_two_links_share_one() { + let (mut m, from_wired, from_wireless) = one_lan(); + let (wired, wireless) = (m.wired, m.wireless); + // A connection from the wireless link, made while the wired one is down. + m.shard.link_down(m.now, wired).unwrap(); + let flow = m.shard.connect(m.now, None, far(80)).unwrap(); + m.shard.receive(m.now, wireless, &router_reply(LAN_GATEWAY, (WIRELESS_MAC, LAN_WIRELESS))); + let mut syn = None; + for _ in 0..2 { + m.shard.transmit(m.now, usize::MAX, |iface, bytes| { + if iface == wireless && bytes.len() >= 54 && bytes[23] == 6 { + syn = Some((u16::from_be_bytes([bytes[34], bytes[35]]), u32::from_be_bytes([bytes[38], bytes[39], bytes[40], bytes[41]]))); + } + }); + m.shard.receive(m.now, wireless, &router_reply(LAN_GATEWAY, (WIRELESS_MAC, LAN_WIRELESS))); + } + let (port, iss) = syn.expect("its SYN, by the wireless link"); + m.shard.link_up(m.now, wired).unwrap(); + m.shard.receive(m.now, wireless, &segment(WIRELESS_MAC, (FAR, 80), (LAN_WIRELESS, port), 7_000, iss.wrapping_add(1), 0x12)); + datagrams(&mut m); + assert!(m.shard.status(flow).is_ok_and(|s| s.state == toyos_net_tcp::State::Established), "the premise: connected"); + both_stale(&mut m, from_wired, from_wireless); + // Two bytes sent and acknowledged: SND.UNA advances, and the advice is the wireless link's. + assert_eq!(m.shard.send(m.now, flow, b"hi"), Ok(2)); + datagrams(&mut m); + m.shard.receive(m.now, wireless, &segment(WIRELESS_MAC, (FAR, 80), (LAN_WIRELESS, port), 7_001, iss.wrapping_add(3), 0x10)); + assert_eq!(gateway_state(&m, wireless), Some("reachable"), "the flow's own gateway entry, confirmed"); + assert_eq!(gateway_state(&m, wired), Some("stale"), "the other link's entry for the same address, untouched"); +} diff --git a/system/netstack/src/main.rs b/system/netstack/src/main.rs index b42f6a36e4e..65702ac6162 100644 --- a/system/netstack/src/main.rs +++ b/system/netstack/src/main.rs @@ -69,8 +69,9 @@ const MAX_LINKS: u32 = 8; /// without them. const INSPECT_BOUND: Duration = Duration::from_secs(1); -/// Drivers' connections that have not said hello, watched at once. -const MAX_PENDING_LINKS: u32 = 2; +/// Drivers' connections that have not said hello, watched at once: every +/// link's driver may be starting together. +const MAX_PENDING_LINKS: u32 = MAX_LINKS; /// The name this machine asks its network to record for it, and answers to on /// it as `.local` once no other host does. One name, because there is @@ -342,13 +343,14 @@ fn answer_inspect(request: &Request, links: &[Held], counters: &[(IfIndex, Count request.client.snapshot(&encoded); } -/// Open the link `asked` asks for on the node's `iface`, and say so. -fn open(asked: Asked, iface: IfIndex, groups: &[Group]) -> Option { +/// Open the link `asked` asks for on the node's `iface`, the `n`th link of +/// the boot, and say so. +fn open(asked: Asked, iface: IfIndex, n: usize, groups: &[Group]) -> Option { let label = asked.hello.label.clone(); let opened = Session::open(asked.hello, iface, asked.conn, asked.region, groups); match &opened { - Some(_) => say!("netstack: {label}'s link is open"), - None => say!("netstack: {label} went before its link was opened"), + Some(_) => say!("netstack: {label}'s link is open, as link {n}"), + None => say!("netstack: {label} went before its link {n} was opened"), } opened } @@ -408,7 +410,7 @@ fn hellos(asked: Vec, links: &mut Vec, node: &mut Node, now: Instan let iface = node.add_link(now, asked.mac, rank(asked.hello.medium), draw); let label = asked.hello.label.clone(); let mac = asked.mac; - let session = open(asked, iface, groups); + let session = open(asked, iface, links.len(), groups); links.push(Held { iface, mac, label, session, faulted: false }); continue; }; @@ -430,7 +432,7 @@ fn hellos(asked: Vec, links: &mut Vec, node: &mut Node, now: Instan } } held.label = asked.hello.label.clone(); - held.session = open(asked, held.iface, groups); + held.session = open(asked, held.iface, at, groups); } } @@ -572,7 +574,9 @@ fn main() { } } - let label = |iface: IfIndex| held.iter().find(|h| h.iface == iface).map_or_else(String::new, |h| h.label.clone()); + let label = |iface: IfIndex| { + held.iter().position(|h| h.iface == iface).map_or_else(String::new, |n| format!("link {n} ({})", held[n].label)) + }; sockets.settle(&mut node, label); // Before anything is served: the lease is what gives this machine an // address, a route and its resolvers. diff --git a/system/netstack/udp/src/lib.rs b/system/netstack/udp/src/lib.rs index 4190ee9fa1b..9458693a07e 100644 --- a/system/netstack/udp/src/lib.rs +++ b/system/netstack/udp/src/lib.rs @@ -218,9 +218,9 @@ struct Queued { payload: Vec, /// The interface the send named, which alone it may leave by. iface: Option, - /// The next hop whose link address it waits for, or the hop of a link with no room: not - /// offered until [`Udp::wake`] names it. - waits: Option, + /// The next hop, and its interface, whose link address it waits for, or the hop of a link + /// with no room: not offered until [`Udp::wake`] names both. + waits: Option<(IfIndex, Ipv4Addr)>, } impl Queued { @@ -278,9 +278,10 @@ pub enum Sender { pub enum Offer { /// It left [udp]: in a frame, or refused by [ip], which counted it and told its flow. Taken, - /// The link address of this next hop is being asked for: the datagram stays with its sender - /// until [`Udp::wake`] or [`Udp::fail`] names the hop. - Waits(Ipv4Addr), + /// The link address of this next hop on this interface is being asked for, or the interface + /// has no room: the datagram stays with its sender until [`Udp::wake`] or [`Udp::fail`] names + /// the hop on its interface. + Waits { iface: IfIndex, hop: Ipv4Addr }, } /// What [`Udp::serve`] handed the caller. @@ -795,7 +796,7 @@ impl Udp { taken = Some(at); break; } - Offer::Waits(next_hop) => queued.waits = Some(next_hop), + Offer::Waits { iface, hop } => queued.waits = Some((iface, hop)), } } let taken = taken.and_then(|at| queue.remove(at)).is_some(); @@ -809,14 +810,15 @@ impl Udp { } } - /// [ip] gave `next_hop` up: every datagram that waited for it is dropped and counted, and - /// the connected socket it left from is told. - pub fn fail(&mut self, next_hop: Ipv4Addr) { + /// [ip] gave `next_hop` on `iface` up: every datagram that waited for it there is dropped + /// and counted, and the connected socket it left from is told. One waiting on the same + /// address on another link stays. + pub fn fail(&mut self, iface: IfIndex, next_hop: Ipv4Addr) { let mut lost = Vec::new(); let queues = self.slots.iter_mut().filter_map(|slot| slot.socket.as_mut()).map(|socket| &mut socket.tx).chain([&mut self.closed]); for queue in queues { queue.retain(|queued| { - let waited = queued.waits == Some(next_hop); + let waited = queued.waits == Some((iface, next_hop)); if waited { lost.push(queued.flow()); } @@ -829,9 +831,10 @@ impl Udp { } } - /// `next_hop`'s link address is known: the datagrams that wait for it are offered again. - pub fn wake(&mut self, next_hop: Ipv4Addr) { - self.rouse(|waits| waits == next_hop); + /// `next_hop`'s link address on `iface` is known, or `iface` has room: the datagrams that + /// wait for it there are offered again. + pub fn wake(&mut self, iface: IfIndex, next_hop: Ipv4Addr) { + self.rouse(|waits| waits == (iface, next_hop)); } /// The routes changed: every waiting datagram's next hop is asked for again. @@ -840,7 +843,7 @@ impl Udp { } /// Ends the wait of every datagram whose next hop `woken` names, and offers its sender. - fn rouse(&mut self, woken: impl Fn(Ipv4Addr) -> bool) { + fn rouse(&mut self, woken: impl Fn((IfIndex, Ipv4Addr)) -> bool) { let Self { slots, closed, closed_offered, eligible, .. } = self; let mut rouse = |queue: &mut VecDeque, offered: &mut bool, sender: Sender| { let mut any = false; diff --git a/system/netstack/udp/tests/common/mod.rs b/system/netstack/udp/tests/common/mod.rs index 1193b7c5e78..e4e33044332 100644 --- a/system/netstack/udp/tests/common/mod.rs +++ b/system/netstack/udp/tests/common/mod.rs @@ -277,8 +277,8 @@ impl U { for event in &events { match event { Event::Unreachable(flow) => self.udp.unreachable(flow), - Event::Resolved { next_hop, .. } => self.udp.wake(*next_hop), - Event::Failed { next_hop, .. } => self.udp.fail(*next_hop), + Event::Resolved { iface, next_hop } => self.udp.wake(*iface, *next_hop), + Event::Failed { iface, next_hop } => self.udp.fail(*iface, *next_hop), _ => {} } } @@ -314,7 +314,7 @@ impl U { framed = true; Offer::Taken } - Ok(Sent::Pending(next_hop)) => Offer::Waits(next_hop), + Ok(Sent::Pending { iface, next_hop }) => Offer::Waits { iface, hop: next_hop }, Ok(Sent::NoRoom { .. }) => unreachable!("every interface has room"), Err(_) => Offer::Taken, } diff --git a/system/sshserver/src/main.rs b/system/sshserver/src/main.rs index 926657ca567..dbeecbd0048 100644 --- a/system/sshserver/src/main.rs +++ b/system/sshserver/src/main.rs @@ -153,8 +153,9 @@ fn main() { let Some(mut listener) = listen() else { return }; // Identity and trust are settled after the bind, so that a machine with no - // NIC still reports the network as the reason it is leaving rather than - // minting a key it will never present. + // netstack reports the network as the reason it is leaving rather than + // minting a key it will never present. A machine with no NIC has a + // netstack, which binds the port: it serves once a link brings one. let host = match host_key() { Ok(key) => Arc::new(key), Err(why) => return println!("sshserver: {why}, exiting"), diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 07c1ee86a0e..d0eac71e113 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -951,6 +951,10 @@ enum Nic { /// QEMU's host is `.2` of. The first is QEMU's default network. pub const TWO_NICS: [(&str, &str); 2] = [("nic0", "10.0.2.0/24"), ("nic1", "10.0.3.0/24")]; +/// Where a guest on [`TWO_NICS`]'s networks reaches [`BootOptions::peer`]: +/// host `.100` of its link's network, port 7. +pub const PEER: (u8, u16) = (100, 7); + /// Everything a profile decides about the machine, in one table. A new /// variant answers every question here or does not compile — which `self != /// Profile::Metal` did the opposite of: it handed anything that was not @@ -1229,6 +1233,11 @@ pub struct BootOptions { /// forwards to the guest's port `.1`: what a peer on the host connects to a /// guest's listener through. Only a profile with a NIC may set it. pub forward: Option<(u16, u16)>, + /// A TCP port of the host's loopback that the guest reaches at [`PEER`] on + /// each of [`TWO_NICS`]'s networks. Only [`Profile::HeadlessTwoNics`] may + /// set it, whose networks are restricted: nothing else of the guest's + /// leaves the host. + pub peer: Option, } impl BootOptions { @@ -1271,6 +1280,7 @@ impl Default for BootOptions { rtc_base: None, bent: None, forward: None, + peer: None, } } } @@ -2523,6 +2533,7 @@ fn qemu_command( // other still creates it after the unit and before everything else. // `iommu_platform` is virtio's own way of asking to be decoded. assert!(options.forward.is_none() || shape.nic != Nic::Absent, "a forwarded port needs a profile with a NIC"); + assert!(options.peer.is_none() || shape.nic == Nic::VirtioPair, "a peer is two NICs' networks'"); let netdev = match options.forward { Some((host, guest)) => format!("user,id=net0,hostfwd=tcp:127.0.0.1:{host}-:{guest}"), None => "user,id=net0".to_string(), @@ -2542,8 +2553,14 @@ fn qemu_command( qemu.arg("-netdev").arg(&netdev).arg("-device").arg("e1000e,netdev=net0,romfile="); } Nic::VirtioPair => { + assert!(options.forward.is_none(), "two NICs' networks forward nothing in; a peer is `peer`'s"); for (n, (id, net)) in TWO_NICS.iter().enumerate() { - let netdev = if n == 0 { netdev.clone() } else { format!("user,id=net{n},net={net}") }; + // `restrict=on`: the guest reaches nothing past QEMU but the peer. + let mut netdev = format!("user,id=net{n},net={net},restrict=on"); + if let Some(port) = options.peer { + let base = net.split('/').next().and_then(|b| b.rsplit_once('.')).map(|(prefix, _)| prefix).expect("a network's address"); + netdev.push_str(&format!(",guestfwd=tcp:{base}.{}:{}-tcp:127.0.0.1:{port}", PEER.0, PEER.1)); + } qemu.arg("-netdev").arg(&netdev).arg("-device").arg(format!( "virtio-net-pci-non-transitional,netdev=net{n},id={id},mac=52:54:00:12:34:5{}{platform}", 6 + n diff --git a/tests/guest/rust-tests/src/bin/link_failover.rs b/tests/guest/rust-tests/src/bin/link_failover.rs index da1b47d7165..07609bd2d0d 100644 --- a/tests/guest/rust-tests/src/bin/link_failover.rs +++ b/tests/guest/rust-tests/src/bin/link_failover.rs @@ -5,9 +5,11 @@ //! this job waits for both links' leases, then sends datagrams to an address //! off both networks: they leave by the link netstack's default route takes, //! which netstack's own count of each link's frames (`inspect net.link..sent`) -//! names. It asks the harness, on a stream over the other link, to set the -//! first link's NIC down; the next datagrams leave by the other link and none -//! by the first. Asked to set it up again, the datagrams return to it. +//! names. It asks the harness, on a stream over the other link to the peer +//! QEMU forwards each network's `.100:7` to, to set the first link's NIC +//! down; the next datagrams leave by the other link and none by the first. +//! Asked to set it up again, the datagrams return to it. The networks are +//! restricted, so the datagrams leave no further than QEMU. use std::collections::BTreeMap; use std::io::{BufRead, BufReader, Write}; @@ -78,10 +80,6 @@ fn carried(by: [u64; 2]) -> (usize, usize) { } fn main() { - let port: u16 = std::env::args() - .nth(1) - .and_then(|port| port.parse().ok()) - .expect("link_failover: the harness's port is this job's argument"); let snap = until("both links' leases", |snap| { snap.get("net.links") == Some(&Value::U64(2)) && (0..2).all(|n| snap.get(&format!("net.link.{n}.lease.held")) == Some(&Value::Bool(true))) }); @@ -98,9 +96,9 @@ fn main() { let (first, second) = carried(round(&socket)); println!(" PASS: a round left by link {first}, on {}", networks[first]); - // The harness, on a stream over the second link: its network's own host. + // The harness, on a stream over the second link: its network's `.100:7`. let [a, b, c, _] = networks[second].octets(); - let mut control = TcpStream::connect(SocketAddr::from((Ipv4Addr::new(a, b, c, 2), port))).expect("link_failover: a stream to the harness over the second link"); + let mut control = TcpStream::connect(SocketAddr::from((Ipv4Addr::new(a, b, c, 100), 7))).expect("link_failover: a stream to the harness over the second link"); control.set_read_timeout(Some(BOUND)).expect("link_failover: a read timeout"); let mut answers = BufReader::new(control.try_clone().expect("link_failover: the stream's reader")); let mut ask = |word: &str| { diff --git a/tests/guest/rust-tests/src/bin/netstack_answers.rs b/tests/guest/rust-tests/src/bin/netstack_answers.rs deleted file mode 100644 index aad6f6498d3..00000000000 --- a/tests/guest/rust-tests/src/bin/netstack_answers.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! One request netstack answers: a UDP bind on any address. netstack serves -//! once a NIC driver's link is open, which the driver opens after its card's -//! bring-up, so the answer follows every line that bring-up logged. Exits 0 -//! on the bind, and 1 naming the refusal. - -use std::net::{Ipv4Addr, UdpSocket}; -use std::process::ExitCode; - -fn main() -> ExitCode { - match UdpSocket::bind((Ipv4Addr::UNSPECIFIED, 0)) { - Ok(socket) => { - println!("netstack_answers: bound {:?}", socket.local_addr()); - ExitCode::SUCCESS - } - Err(e) => { - println!("netstack_answers: the bind was refused: {e}"); - ExitCode::FAILURE - } - } -} diff --git a/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs b/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs index 997f013eade..5616f971b1e 100644 --- a/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs +++ b/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs @@ -4,10 +4,10 @@ //! `/system/bin/sshserver` has one clean exit for a machine with no network and it is keyed //! on an error kind: `ErrorKind::NotConnected`, which is what //! `NetError::NetstackNotFound` becomes. Anything else panics, by design — a -//! machine that *has* a NIC and cannot bind must be loud. On a NIC-less machine -//! netstack prints its line and exits, and whether sshserver took the quiet arm or put a -//! tokio backtrace across the boot depended on which side of netstack's teardown -//! its bind landed. +//! machine that *has* a NIC and cannot bind must be loud. netstack that exits, as it +//! once did on a NIC-less machine, can do so under a bind, and whether sshserver took +//! the quiet arm or put a tokio backtrace across the boot depended on which side of +//! netstack's teardown its bind landed. //! //! **The race is not staged here. The sequence is.** What made the defect hard //! to see is that it is a handful of instructions wide in a real boot; what diff --git a/tests/toyos.rs b/tests/toyos.rs index 635655f8927..bb975c2a7c0 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -139,12 +139,9 @@ const RUST_SKIP: &[&str] = &[ // It measures the T14's I219, which no guest has: the // `i219_messages_on_its_msi_vector` metal row runs it. "i219_msi", - // It waits for netstack to answer, which follows the I219's bring-up: the - // `i219_leaves_ulp_unarmed` metal row runs it on tests/images/benchcase.toml. - "netstack_answers", - // It waits for a lease from the bench's router over the T14's I219: the - // `i219_carries_netstacks_link` metal row runs it on - // tests/images/benchcase.toml. + // It waits for a lease from the bench's router over the T14's I219, which + // follows the card's bring-up: the `i219_carries_netstacks_link` and + // `i219_leaves_ulp_unarmed` metal rows run it on tests/images/benchcase.toml. "lan_lease", // Its product is the T14's counters across an idle span and a spin on // every CPU, which the `counters` metal row judges; on a guest it would be @@ -684,10 +681,10 @@ const MACHINE_TESTS: &[&str] = &[ // when it is up. Two drivers' links, a NIC's carrier and netstack's // routes cross three processes and two devices, which the node's and the // shard's host tests of rank, failover and return hold below the drivers. - // QEMU x86-64 (`qemu::SUITE_ARCH`), on two user networks; the datagrams go - // to RFC 5737's TEST-NET-1, which answers nothing, and the harness is the - // peer on the host's loopback, so no public network. Order and counts - // only. + // QEMU x86-64 (`qemu::SUITE_ARCH`), on two restricted user networks: the + // datagrams to RFC 5737's TEST-NET-1 leave no further than QEMU, and the + // harness is the peer each network forwards `.100:7` to on the host's + // loopback, so no public network. Order and counts only. "traffic_leaves_by_the_second_link_while_the_first_is_down", // Login sessions: their owner parents, signs out and ends with each, and // the machine's reboot waits for them until forced, which only processes @@ -1824,18 +1821,17 @@ const OSBENCH_INTERNET: &[metal::Arm] = &[metal::Arm { ..metal::once("netbenchcase", "tests/images/benchcase.toml", &[], &["osbench_internet"]) }]; -/// [`BENCHCASE`]'s boot, with two jobs: netstack answering, which it does only -/// once the card's bring-up has logged, and its lease landing. Without them -/// the list's `reboot` lands while the driver is still reaching the PHY. -const BENCHCASE_ALONE: &[metal::Arm] = - &[metal::once("benchcase", "tests/images/benchcase.toml", &[], &["test_rs_netstack_answers", "test_rs_lan_lease"])]; +/// [`BENCHCASE`]'s boot, with one job: the lease landing, which follows the +/// card's bring-up and every line it logged. Without it the list's `reboot` +/// lands while the driver is still reaching the PHY. +const BENCHCASE_ALONE: &[metal::Arm] = &[metal::once("benchcase", "tests/images/benchcase.toml", &[], &["test_rs_lan_lease"])]; /// The I219 driver's account of its part's start: what its wake found of /// Ultra Low Power and disarmed, and the configuration its bring-up read back /// unarmed. Both lines are printed, because what was found is what the next /// boot's diagnosis reads. fn i219_leaves_ulp_unarmed(boot: &metal::Readback) -> Result<(), String> { - boot.job_passed("test_rs_netstack_answers")?; + boot.job_passed("test_rs_lan_lease")?; let log = boot.log(); let woke = log.must_say("i219: before the reset the PHY was asked:")?; let left = log.must_say("it is left with ULP configuration")?; @@ -5806,10 +5802,6 @@ fn peer_verdict(heard: std::thread::JoinHandle>) -> String } } -/// `test_rs_link_restart` on `tests/images/linkcase.toml`'s boot: the peer -/// answers each line the job writes on one connection, the one before the -/// job ends its driver and the one after it starts another, then reads the -/// job's FIN and answers it with its own. /// `test_rs_link_failover` on two virtio NICs: the harness, asked over the /// second link's network, sets the first link's NIC down and then up. fn traffic_leaves_by_the_second_link_while_the_first_is_down() -> Result<(), String> { @@ -5820,11 +5812,11 @@ fn traffic_leaves_by_the_second_link_while_the_first_is_down() -> Result<(), Str let tests = compile::repo_root().join("tests/guest/rust-tests"); let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &tests, JOB); let config = compile::repo_root().join("tests/images/netcase.toml"); - let options = BootOptions { profile: qemu::Profile::HeadlessTwoNics, qmp: true, ..BootOptions::default() }; - let mut qemu = QemuInstance::boot_with_options(&config, &[], &[(JOB.to_string(), bin)], options); - serial::Serial::boot(&qemu).must_be_clean()?; let peer = std::net::TcpListener::bind("127.0.0.1:0").map_err(|e| format!("no loopback listener: {e}"))?; let port = peer.local_addr().map_err(|e| format!("the listener's port: {e}"))?.port(); + let options = BootOptions { profile: qemu::Profile::HeadlessTwoNics, qmp: true, peer: Some(port), ..BootOptions::default() }; + let mut qemu = QemuInstance::boot_with_options(&config, &[], &[(JOB.to_string(), bin)], options); + serial::Serial::boot(&qemu).must_be_clean()?; let socket = qemu.qmp_socket().to_path_buf(); let heard = std::thread::spawn(move || -> Result<(), String> { let (conn, _) = peer.accept().map_err(|e| format!("the guest's connect: {e}"))?; @@ -5847,7 +5839,7 @@ fn traffic_leaves_by_the_second_link_while_the_first_is_down() -> Result<(), Str } Ok(()) }); - let result = qemu.run_test(&format!("test_rs_{JOB} {port}"), CEILING); + let result = qemu.run_test(&format!("test_rs_{JOB}"), CEILING); if let Some(why) = &result.error { return Err(format!("{why}\n{}\nthe job said:\n{}", peer_verdict(heard), result.stdout)); } @@ -5861,6 +5853,10 @@ fn traffic_leaves_by_the_second_link_while_the_first_is_down() -> Result<(), Str Ok(()) } +/// `test_rs_link_restart` on `tests/images/linkcase.toml`'s boot: the peer +/// answers each line the job writes on one connection, the one before the +/// job ends its driver and the one after it starts another, then reads the +/// job's FIN and answers it with its own. fn a_link_ends_under_a_held_stream_and_comes_back() -> Result<(), String> { use std::io::{BufRead, BufReader, Write}; const JOB: &str = "link_restart"; From 93715e77ad93ca7cc170dfeda8db2e66c0366243 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 00:20:04 +0100 Subject: [PATCH 7/9] review r1 of #1122: the two-NIC networks stay unrestricted With restrict=on the failover job's first datagram off both networks was refused for want of a route, and the default route is what the test is of; the harness stays the peer at each network's QEMU host address. Co-Authored-By: Claude Opus 5.5 --- tests/common/qemu.rs | 19 +------------------ .../guest/rust-tests/src/bin/link_failover.rs | 16 +++++++++------- tests/toyos.rs | 18 ++++++++++-------- 3 files changed, 20 insertions(+), 33 deletions(-) diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index d0eac71e113..07c1ee86a0e 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -951,10 +951,6 @@ enum Nic { /// QEMU's host is `.2` of. The first is QEMU's default network. pub const TWO_NICS: [(&str, &str); 2] = [("nic0", "10.0.2.0/24"), ("nic1", "10.0.3.0/24")]; -/// Where a guest on [`TWO_NICS`]'s networks reaches [`BootOptions::peer`]: -/// host `.100` of its link's network, port 7. -pub const PEER: (u8, u16) = (100, 7); - /// Everything a profile decides about the machine, in one table. A new /// variant answers every question here or does not compile — which `self != /// Profile::Metal` did the opposite of: it handed anything that was not @@ -1233,11 +1229,6 @@ pub struct BootOptions { /// forwards to the guest's port `.1`: what a peer on the host connects to a /// guest's listener through. Only a profile with a NIC may set it. pub forward: Option<(u16, u16)>, - /// A TCP port of the host's loopback that the guest reaches at [`PEER`] on - /// each of [`TWO_NICS`]'s networks. Only [`Profile::HeadlessTwoNics`] may - /// set it, whose networks are restricted: nothing else of the guest's - /// leaves the host. - pub peer: Option, } impl BootOptions { @@ -1280,7 +1271,6 @@ impl Default for BootOptions { rtc_base: None, bent: None, forward: None, - peer: None, } } } @@ -2533,7 +2523,6 @@ fn qemu_command( // other still creates it after the unit and before everything else. // `iommu_platform` is virtio's own way of asking to be decoded. assert!(options.forward.is_none() || shape.nic != Nic::Absent, "a forwarded port needs a profile with a NIC"); - assert!(options.peer.is_none() || shape.nic == Nic::VirtioPair, "a peer is two NICs' networks'"); let netdev = match options.forward { Some((host, guest)) => format!("user,id=net0,hostfwd=tcp:127.0.0.1:{host}-:{guest}"), None => "user,id=net0".to_string(), @@ -2553,14 +2542,8 @@ fn qemu_command( qemu.arg("-netdev").arg(&netdev).arg("-device").arg("e1000e,netdev=net0,romfile="); } Nic::VirtioPair => { - assert!(options.forward.is_none(), "two NICs' networks forward nothing in; a peer is `peer`'s"); for (n, (id, net)) in TWO_NICS.iter().enumerate() { - // `restrict=on`: the guest reaches nothing past QEMU but the peer. - let mut netdev = format!("user,id=net{n},net={net},restrict=on"); - if let Some(port) = options.peer { - let base = net.split('/').next().and_then(|b| b.rsplit_once('.')).map(|(prefix, _)| prefix).expect("a network's address"); - netdev.push_str(&format!(",guestfwd=tcp:{base}.{}:{}-tcp:127.0.0.1:{port}", PEER.0, PEER.1)); - } + let netdev = if n == 0 { netdev.clone() } else { format!("user,id=net{n},net={net}") }; qemu.arg("-netdev").arg(&netdev).arg("-device").arg(format!( "virtio-net-pci-non-transitional,netdev=net{n},id={id},mac=52:54:00:12:34:5{}{platform}", 6 + n diff --git a/tests/guest/rust-tests/src/bin/link_failover.rs b/tests/guest/rust-tests/src/bin/link_failover.rs index 07609bd2d0d..da1b47d7165 100644 --- a/tests/guest/rust-tests/src/bin/link_failover.rs +++ b/tests/guest/rust-tests/src/bin/link_failover.rs @@ -5,11 +5,9 @@ //! this job waits for both links' leases, then sends datagrams to an address //! off both networks: they leave by the link netstack's default route takes, //! which netstack's own count of each link's frames (`inspect net.link..sent`) -//! names. It asks the harness, on a stream over the other link to the peer -//! QEMU forwards each network's `.100:7` to, to set the first link's NIC -//! down; the next datagrams leave by the other link and none by the first. -//! Asked to set it up again, the datagrams return to it. The networks are -//! restricted, so the datagrams leave no further than QEMU. +//! names. It asks the harness, on a stream over the other link, to set the +//! first link's NIC down; the next datagrams leave by the other link and none +//! by the first. Asked to set it up again, the datagrams return to it. use std::collections::BTreeMap; use std::io::{BufRead, BufReader, Write}; @@ -80,6 +78,10 @@ fn carried(by: [u64; 2]) -> (usize, usize) { } fn main() { + let port: u16 = std::env::args() + .nth(1) + .and_then(|port| port.parse().ok()) + .expect("link_failover: the harness's port is this job's argument"); let snap = until("both links' leases", |snap| { snap.get("net.links") == Some(&Value::U64(2)) && (0..2).all(|n| snap.get(&format!("net.link.{n}.lease.held")) == Some(&Value::Bool(true))) }); @@ -96,9 +98,9 @@ fn main() { let (first, second) = carried(round(&socket)); println!(" PASS: a round left by link {first}, on {}", networks[first]); - // The harness, on a stream over the second link: its network's `.100:7`. + // The harness, on a stream over the second link: its network's own host. let [a, b, c, _] = networks[second].octets(); - let mut control = TcpStream::connect(SocketAddr::from((Ipv4Addr::new(a, b, c, 100), 7))).expect("link_failover: a stream to the harness over the second link"); + let mut control = TcpStream::connect(SocketAddr::from((Ipv4Addr::new(a, b, c, 2), port))).expect("link_failover: a stream to the harness over the second link"); control.set_read_timeout(Some(BOUND)).expect("link_failover: a read timeout"); let mut answers = BufReader::new(control.try_clone().expect("link_failover: the stream's reader")); let mut ask = |word: &str| { diff --git a/tests/toyos.rs b/tests/toyos.rs index bb975c2a7c0..0d56ac2e944 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -681,10 +681,12 @@ const MACHINE_TESTS: &[&str] = &[ // when it is up. Two drivers' links, a NIC's carrier and netstack's // routes cross three processes and two devices, which the node's and the // shard's host tests of rank, failover and return hold below the drivers. - // QEMU x86-64 (`qemu::SUITE_ARCH`), on two restricted user networks: the - // datagrams to RFC 5737's TEST-NET-1 leave no further than QEMU, and the - // harness is the peer each network forwards `.100:7` to on the host's - // loopback, so no public network. Order and counts only. + // QEMU x86-64 (`qemu::SUITE_ARCH`), on two user networks; the datagrams go + // to RFC 5737's TEST-NET-1, which answers nothing, and the harness is the + // peer on the host's loopback, so no public network. The networks are not + // `restrict=on`: with it the job's first datagram off both networks was + // refused for want of a route, and the default route is what is tested. + // Order and counts only. "traffic_leaves_by_the_second_link_while_the_first_is_down", // Login sessions: their owner parents, signs out and ends with each, and // the machine's reboot waits for them until forced, which only processes @@ -5812,11 +5814,11 @@ fn traffic_leaves_by_the_second_link_while_the_first_is_down() -> Result<(), Str let tests = compile::repo_root().join("tests/guest/rust-tests"); let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &tests, JOB); let config = compile::repo_root().join("tests/images/netcase.toml"); - let peer = std::net::TcpListener::bind("127.0.0.1:0").map_err(|e| format!("no loopback listener: {e}"))?; - let port = peer.local_addr().map_err(|e| format!("the listener's port: {e}"))?.port(); - let options = BootOptions { profile: qemu::Profile::HeadlessTwoNics, qmp: true, peer: Some(port), ..BootOptions::default() }; + let options = BootOptions { profile: qemu::Profile::HeadlessTwoNics, qmp: true, ..BootOptions::default() }; let mut qemu = QemuInstance::boot_with_options(&config, &[], &[(JOB.to_string(), bin)], options); serial::Serial::boot(&qemu).must_be_clean()?; + let peer = std::net::TcpListener::bind("127.0.0.1:0").map_err(|e| format!("no loopback listener: {e}"))?; + let port = peer.local_addr().map_err(|e| format!("the listener's port: {e}"))?.port(); let socket = qemu.qmp_socket().to_path_buf(); let heard = std::thread::spawn(move || -> Result<(), String> { let (conn, _) = peer.accept().map_err(|e| format!("the guest's connect: {e}"))?; @@ -5839,7 +5841,7 @@ fn traffic_leaves_by_the_second_link_while_the_first_is_down() -> Result<(), Str } Ok(()) }); - let result = qemu.run_test(&format!("test_rs_{JOB}"), CEILING); + let result = qemu.run_test(&format!("test_rs_{JOB} {port}"), CEILING); if let Some(why) = &result.error { return Err(format!("{why}\n{}\nthe job said:\n{}", peer_verdict(heard), result.stdout)); } From d1e7a1eaf27f9dfc51d66932929a86dd25ab4a78 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 00:39:02 +0100 Subject: [PATCH 8/9] netstack_connect_killed waits for a lease on any link main's new job read net.lease.held, which P5 puts under each link as net.link..lease.held. Co-Authored-By: Claude Opus 5.5 --- tests/guest/rust-tests/src/bin/netstack_connect_killed.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/guest/rust-tests/src/bin/netstack_connect_killed.rs b/tests/guest/rust-tests/src/bin/netstack_connect_killed.rs index 5b756161a36..c504b944a14 100644 --- a/tests/guest/rust-tests/src/bin/netstack_connect_killed.rs +++ b/tests/guest/rust-tests/src/bin/netstack_connect_killed.rs @@ -49,7 +49,10 @@ fn main() { } fn test() { - until(|snapshot| snapshot.get("net.lease.held") == Some(&Value::Bool(true)), "netstack held a lease"); + until( + |snapshot| snapshot.iter().any(|(key, value)| key.starts_with("net.link.") && key.ends_with(".lease.held") && *value == Value::Bool(true)), + "netstack held a lease on a link", + ); let before = held(&snapshot()); let connect = || Command::new(SELF_PATH).arg("connect").stdin(Stdio::null()).spawn().expect("a connecting child"); let (mut killed, mut survivor) = (connect(), connect()); From e71bcf0376ae5a8a0fef2fee7f72e6d56abd1875 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 02:26:20 +0100 Subject: [PATCH 9/9] review r2 of #1122: the ULP row's comment and the ACD queue tests say what they hold - i219_leaves_ulp_unarmed's comment: it reads the driver's lines once its job has seen the lease, which follows the bring-up. - acd.rs: the helper is `answering()`, if0 holding ANSWERING, whose replies fill if0's own share of the queue; its callers name if0, the comment says so, and the two asserts check the frames that led are ARP replies. - Headers: udp and the shard say a datagram waits on the hop on its interface; name.rs notes the bridged message that crosses as its link loses its address; lease.rs's long line wrapped; the failover comment says it depends on no public network. Co-Authored-By: Claude Opus 5.5 --- system/netstack/ip/tests/acd.rs | 37 +++++++++++++++++++------------ system/netstack/node/src/lease.rs | 7 +++--- system/netstack/node/src/name.rs | 5 ++++- system/netstack/shard/src/lib.rs | 4 ++-- system/netstack/udp/src/lib.rs | 5 +++-- tests/toyos.rs | 4 ++-- 6 files changed, 38 insertions(+), 24 deletions(-) diff --git a/system/netstack/ip/tests/acd.rs b/system/netstack/ip/tests/acd.rs index e3f0de96207..3c6f810bcfe 100644 --- a/system/netstack/ip/tests/acd.rs +++ b/system/netstack/ip/tests/acd.rs @@ -7,7 +7,7 @@ use std::time::Duration; use common::*; use toyos_net_ip::limits::acd::{ANNOUNCE_WAIT, PROBE_MAX, PROBE_MIN, PROBE_WAIT}; -use toyos_net_ip::{limits, AddrState, Counter, Event, IfIndex, Instant, Peer, Source}; +use toyos_net_ip::{limits, AddrState, Counter, Event, Instant, Peer, Source}; use toyos_net_wire::ethernet::MacAddr; fn ip4(a: u8, b: u8, c: u8, d: u8) -> Ipv4Addr { @@ -268,21 +268,29 @@ fn s_ip_acd_017_detection_starts_at_the_first_announcement() { assert_eq!(h.count(Counter::AcdDefended), 1); } -/// if0 holding 198.51.100.1/24 and nothing else yet, its control queue for the tests to fill -/// with answers to it: each interface's share of the queue is its own. -fn beside_if1() -> (H, IfIndex) { +/// The address on if0 whose ARP replies fill its share of the control queue. +const ANSWERING: Ipv4Addr = Ipv4Addr::new(198, 51, 100, 1); + +/// if0 holding [`ANSWERING`] and nothing else yet: its share of the control queue, which is +/// its own, for the tests to fill with answers to it. +fn answering() -> H { let mut h = H::raw(); let if0 = h.if0; - h.assign(if0, ip4(198, 51, 100, 1), 24, 0); + h.assign(if0, ANSWERING, 24, 0); h.settle(); - (h, if0) + h +} + +/// An ARP reply's frame: EtherType 0x0806, operation 2. +fn is_arp_reply(frame: &[u8]) -> bool { + frame.get(12..14) == Some(&[0x08, 0x06]) && frame.get(20..22) == Some(&[0, 2]) } #[test] fn s_ip_acd_018_a_full_control_queue_holds_probes_and_announcements() { - let (mut h, if1) = beside_if1(); + let mut h = answering(); let t0 = h.clock(); - h.fill_control_queue(if1, t0, ip4(198, 51, 100, 1)); + h.fill_control_queue(h.if0, t0, ANSWERING); h.ip.add_address(t0, h.if0, A, 24).unwrap(); let at = t0.after(Duration::from_secs(1)); while let Some(d) = h.ip.next_deadline().filter(|d| *d <= at) { @@ -296,7 +304,7 @@ fn s_ip_acd_018_a_full_control_queue_holds_probes_and_announcements() { let mut out = Vec::new(); h.ip.transmit(at, limits::CONTROL_QUEUE, |iface, f| out.push((iface, f.to_vec()))); assert_eq!(out.len(), limits::CONTROL_QUEUE); - assert!(out.iter().all(|(iface, _)| *iface == if1), "the replies queued first"); + assert!(out.iter().all(|(iface, f)| *iface == h.if0 && is_arp_reply(f)), "the replies queued first"); out.clear(); h.ip.transmit(at, 1, |iface, f| out.push((iface, f.to_vec()))); assert_eq!(out, [(h.if0, hex(V_ARP_PROBE))]); @@ -305,12 +313,13 @@ fn s_ip_acd_018_a_full_control_queue_holds_probes_and_announcements() { assert!(h.events.contains(&Event::Verified { iface: h.if0, addr: A })); assert_eq!(h.ip.address(h.if0, A), Some(AddrState::Assigned)); - // A probes with credit until its third probe leaves; then if1 fills the queue, and the first - // announcement it owes waits there, counted toward nothing until it leaves. - let (mut h, if1) = beside_if1(); + // A probes with credit until its third probe leaves; then answers to ANSWERING fill if0's + // share of the queue, and the first announcement A owes waits there, counted toward nothing + // until it leaves. + let mut h = answering(); h.ip.add_address(h.clock(), h.if0, A, 24).unwrap(); let p3 = third_probe(&mut h); - h.fill_control_queue(if1, p3, ip4(198, 51, 100, 1)); + h.fill_control_queue(h.if0, p3, ANSWERING); let verified = p3.after(ANNOUNCE_WAIT); h.ip.fire(verified); h.collect(); @@ -325,7 +334,7 @@ fn s_ip_acd_018_a_full_control_queue_holds_probes_and_announcements() { let mut out = Vec::new(); h.ip.transmit(t1, usize::MAX, |iface, f| out.push((iface, f.to_vec()))); assert_eq!(out.len(), limits::CONTROL_QUEUE + 1, "nothing more was queued"); - assert!(out[..limits::CONTROL_QUEUE].iter().all(|(iface, _)| *iface == if1), "the replies queued first"); + assert!(out[..limits::CONTROL_QUEUE].iter().all(|(iface, f)| *iface == h.if0 && is_arp_reply(f)), "the replies queued first"); assert_eq!(out[limits::CONTROL_QUEUE], (h.if0, hex(V_ARP_ANNOUNCE))); assert!(run_exact(&mut h, t1.after(Duration::from_millis(1_999))).is_empty()); assert_eq!(h.ip.address(h.if0, A), Some(AddrState::Announcing), "ANNOUNCE_INTERVAL runs from the hand-off"); diff --git a/system/netstack/node/src/lease.rs b/system/netstack/node/src/lease.rs index 0da52207877..64d3420028b 100644 --- a/system/netstack/node/src/lease.rs +++ b/system/netstack/node/src/lease.rs @@ -10,9 +10,10 @@ //! report for the link's address under probe, so no address is in use before conflict //! detection ends; //! - the resolvers lookups ask are one link's held lease's: of the links up and holding one, the -//! first by rank and then arrival, and with none up the link asked before ([`Stack::asked`]). The answers they gave go with every write of any -//! link's state, every link that goes down and every change of the link asked: after any of -//! them the next lookup may be asking another network, so none is answered from this one's. +//! first by rank and then arrival, and with none up the link asked before ([`Stack::asked`]). +//! The answers they gave go with every write of any link's state, every link that goes down +//! and every change of the link asked: after any of them the next lookup may be asking another +//! network, so none is answered from this one's. //! //! [ip] reports on no address but those this module added, and one report at a time: each is //! read against its link's state as it stands. diff --git a/system/netstack/node/src/name.rs b/system/netstack/node/src/name.rs index 806ec74fa44..7c00e6c838d 100644 --- a/system/netstack/node/src/name.rs +++ b/system/netstack/node/src/name.rs @@ -9,7 +9,10 @@ //! - **A message from one of this machine's own addresses never reaches a responder**: [ip] //! refuses a datagram whose source any of its interfaces holds (`ip.own-source`), so on one LAN //! two links share, as a wired and a wireless link bridged (§14), this machine hears its own -//! messages from neither and never conflicts with itself. +//! messages from neither and never conflicts with itself. The one exception is a message that +//! crosses the bridge as its sending link loses the address it came from: [ip] no longer holds +//! that source, a millisecond's race, and the other link's responder may read it as another +//! host's. //! - The record is the link's held lease's on a link that is up: its address and prefix are what the //! responder is told, and it is told it has no link from the moment the lease goes or the link //! does. Nothing is probed for, announced or answered for an address still under probe, or on a diff --git a/system/netstack/shard/src/lib.rs b/system/netstack/shard/src/lib.rs index 9a7927d89e7..4b4bf9be3fa 100644 --- a/system/netstack/shard/src/lib.rs +++ b/system/netstack/shard/src/lib.rs @@ -17,8 +17,8 @@ //! refused, that the table has room: a waiting flow costs one question per such change. A UDP //! datagram [ip] does not take, its next hop unresolved, stays in its sender's queue in [udp], //! spending nothing and charged nothing, while the sender's datagrams to other next hops leave: -//! it is handed to [ip] again when [ip] reports that hop resolved or the routes changed, and -//! dropped by [udp] when [ip] reports the hop failed. One [ip] refuses, for want of a route or +//! it is handed to [ip] again when [ip] reports the hop on its interface resolved or the routes +//! changed, and dropped by [udp] when [ip] reports the hop on its interface failed. One [ip] refuses, for want of a route or //! for a next hop it holds failed or has no room for, is [ip]'s to count and report. A flow //! whose route leaves by a link with no room waits on that link as on a next hop, and asks again //! at the next opportunity that link has room in. diff --git a/system/netstack/udp/src/lib.rs b/system/netstack/udp/src/lib.rs index 9458693a07e..11dc4ec3bd1 100644 --- a/system/netstack/udp/src/lib.rs +++ b/system/netstack/udp/src/lib.rs @@ -15,8 +15,9 @@ //! //! **A datagram waits for its next hop where it was accepted.** One the caller answers //! [`Offer::Waits`] stays in its sender's queue, under that queue's bound, and is passed over -//! until [`Udp::wake`] names the hop: the sender's datagrams to other hops leave meanwhile, and -//! those to one hop leave in the order they were accepted. One whose hop [`Udp::fail`] names is +//! until [`Udp::wake`] names the hop on its interface: the sender's datagrams to other hops leave +//! meanwhile, and those to one hop leave in the order they were accepted. One whose hop on its +//! interface [`Udp::fail`] names is //! dropped, counted, and reported to a connected socket; nothing is kept for another try. //! //! **Refusals are values.** Every refusal is a named [`Counter`] and the [`Error`] the call diff --git a/tests/toyos.rs b/tests/toyos.rs index 0d56ac2e944..6e76b1fdeff 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -683,7 +683,7 @@ const MACHINE_TESTS: &[&str] = &[ // shard's host tests of rank, failover and return hold below the drivers. // QEMU x86-64 (`qemu::SUITE_ARCH`), on two user networks; the datagrams go // to RFC 5737's TEST-NET-1, which answers nothing, and the harness is the - // peer on the host's loopback, so no public network. The networks are not + // peer on the host's loopback, so it depends on no public network. The networks are not // `restrict=on`: with it the job's first datagram off both networks was // refused for want of a route, and the default route is what is tested. // Order and counts only. @@ -1232,7 +1232,7 @@ const METAL: &[(&str, metal::Metal)] = &[ // full reset, which the stub only assumes. The T14 only: the AMD bench // has no card ToyOS drives. It rides `osbench`'s boot, whose i219 row // drives the card, and reads two of the driver's lines once its job - // has had netstack answer, which it does only after the bring-up. + // has seen the lease, which follows the bring-up. "i219_leaves_ulp_unarmed", metal::Metal { arms: BENCHCASE_ALONE, judge: |b| i219_leaves_ulp_unarmed(b[0]) }, ),