diff --git a/Cargo.lock b/Cargo.lock index 19f6ddbcba9..60cf8ff4a30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3908,6 +3908,7 @@ version = "0.1.0" dependencies = [ "etherparse", "pcap-file", + "toyos-net-ip", "toyos-net-shard", "toyos-net-tcp", "toyos-net-wire", diff --git a/images/system.toml b/images/system.toml index 9e2ef83dc4b..623058a9a4c 100644 --- a/images/system.toml +++ b/images/system.toml @@ -132,9 +132,8 @@ drives = ["pci:04:03:*"] receives = ["sound-output"] syscap = ["rt"] -# The network stack, over the link a NIC driver brings on `link`; it claims no -# device. It serves once the first driver says hello, and leaves a machine -# none says hello on within 30 s. +# The network stack, over every link a NIC driver brings on `link`; it claims +# no device. It serves from the start, links or none. [programs.netstack] service = true serves = ["netstack", "link"] diff --git a/issues/first-match-selection-that-remains.md b/issues/first-match-selection-that-remains.md index e6e33da4b1c..baad4d064ad 100644 --- a/issues/first-match-selection-that-remains.md +++ b/issues/first-match-selection-that-remains.md @@ -12,10 +12,11 @@ does so visibly. Two do, and both are deliberate: - **NVMe.** blockd serves one controller, the first its row's claim names (`userland/blockd`); a machine with two NVMe drives loses the second. Making this an enumerate-all is a block-service change, not a PCI one. -- **The four virtio drivers.** Each takes the first device with its - (vendor, device) pair. A second NIC or a second GPU would be dropped. These - are QEMU-only devices — no virtio function appears on the T14 — so the - exposure is a test-shape one, and no profile declares two of anything virtio. +- **The virtio drivers but the NIC's.** Each takes the first device with its + (vendor, device) pair, and a second GPU would be dropped. These are + QEMU-only devices — no virtio function appears on the T14 — so the exposure + is a test-shape one. The NIC's driver is one process per function, and + netstack runs every link they bring: `Profile::HeadlessTwoNics` has two. Neither is a defect today. Both become one the moment a second such device is reachable, and the enumerate-all they would need now exists. diff --git a/issues/netstack-drives-one-card.md b/issues/netstack-drives-one-card.md deleted file mode 100644 index 7843af9cb34..00000000000 --- a/issues/netstack-drives-one-card.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-12 ---- - -# netstack drives one card - -netstack opens the first link a NIC driver brings, and no other: a hello with -another address, or with that address while its link is alive, is refused -`OneLink` (`system/netstack/src/main.rs`'s loop). -Its node is one interface with one DHCP client, made with the first link's -address (`system/netstack/node/src/lib.rs`). So a machine with a wired and a -wireless link uses only one of them, whichever driver says hello first. - -Because the node needs that address, netstack serves nothing before a -driver's hello: every client waits for it, and on a machine with no NIC -driver for 30 s (`LINK_BOUND`, `system/netstack/src/main.rs`), after which -netstack leaves and a client hears it gone. [ip] already -routes over several interfaces, the lower index winning a tie -(`system/netstack/ip/src/lib.rs`). - -The owner ruled for several interfaces over one chosen by the manifest. He -also ruled that "netstack runs every link a driver brings, wired first, with -failover" (#876, issuecomment-6120490125, ruling 3). - -That reverses part of `issues/first-match-selection-that-remains.md` -(`kind: rejected`), whose virtio bullet says a second virtio NIC "would be -dropped" and that "Neither is a defect today". Under ruling 3 a second NIC -dropped is this defect. - -Owner: P5 of `issues/there-is-no-wifi.md`. The drivers are out: each is a -process of its own over a link (`toyos-link`). - -**Exit**: -- one interface per link, each with its own DHCP client, lease and mDNS - responder, under one host name, and a node that serves before any link, - so `LINK_BOUND` goes; -- links ranked wired before wireless, then by arrival; -- a guest with two virtio NICs in which the first's link is set off over - QMP, and traffic leaves on the second; -- the NIC taken out of the virtio bullet of - `issues/first-match-selection-that-remains.md`. diff --git a/issues/there-is-no-wifi.md b/issues/there-is-no-wifi.md index ffc279aa16d..66b0720296e 100644 --- a/issues/there-is-no-wifi.md +++ b/issues/there-is-no-wifi.md @@ -216,15 +216,21 @@ stage 3 of `issues/the-lan-is-not-yet-production-grade.md`, whole. second each way full rate needs (https://github.com/ToyOSOrg/evidence/blob/4a8d02243d626df929fe05365e991873574e0dc2/s2/README.md). -**P5, netstack over several links.** Owner: the orchestrator; its defect is -`issues/netstack-drives-one-card.md`. Depends on: P4. +**P5, netstack over several links: built.** Owner: the orchestrator. +Depends on: P4. - **Exit:** - host tests of rank (wired before wireless, then arrival), failover and return, resolvers, mDNS per link, and a flow bound to a lost address; - a guest with two virtio NICs, in which the first's link is set off over QMP and traffic leaves on the second; - - `issues/netstack-drives-one-card.md` closed, and the NIC taken out of - the virtio bullet of `issues/first-match-selection-that-remains.md`. + - its defect closed, and the NIC taken out of the virtio bullet of + `issues/first-match-selection-that-remains.md`. +- **What it is.** One stack over every link a driver brings, up to eight: + each an interface of `ip`'s with a rank (a hello says Wired or Wireless), + its own lease, DHCP client and mDNS responder under one host name; lookups + ask the resolvers of the first link up holding a lease. A connection whose + address goes ends `AddressGone`; one whose link only goes down keeps it. + netstack serves from the start, so `LINK_BOUND` and `OneLink` are gone. - Its T14 row, both links leasing with the wired one the default route, is W6's: it needs a radio. diff --git a/issues/two-links-leased-one-address-the-second-holds-none.md b/issues/two-links-leased-one-address-the-second-holds-none.md new file mode 100644 index 00000000000..17f3e26da1a --- /dev/null +++ b/issues/two-links-leased-one-address-the-second-holds-none.md @@ -0,0 +1,27 @@ +--- +status: open +kind: defect +opened: 2026-11-01 +--- + +# Two links leased one address: the second never holds one + +netstack runs every link a driver brings (`issues/there-is-no-wifi.md`, P5), +and [ip] keeps an address unique across its interfaces: a second interface +adding an address the first holds is refused `addr.duplicate`. Two +separate networks can lease the same address, two home routers each handing +out 192.168.1.10, and then the second link's probe is refused, its client +counts `node.address-refused` and starts its exchange over, and the same +server offers it the same address again: the second link never holds an +address, its client asks for the boot, and its DHCP lines repeat. + +Nothing breaks the first link, and the uniqueness rule is what keeps +`toyos-net-tcp`'s `lose_address` and the strong host model's source +selection sound: one address names one interface. + +**Owner**: the orchestrator. + +**Exit**: the second link's client declines the address (RFC 2131 §3.1, a +DECLINE) and asks for another, as it would for one ACD found in use, so that +the second link leases a different address where its server has one; with a +host test of two links offered one address. diff --git a/lib/link/src/driver.rs b/lib/link/src/driver.rs index 4216b88ff89..445fe45b1c4 100644 --- a/lib/link/src/driver.rs +++ b/lib/link/src/driver.rs @@ -27,11 +27,14 @@ use crate::ends::{Driver, Fault}; use crate::layout::{RX_DEPTH, SLOT_BYTES, TX_DEPTH}; use crate::region::Region; use crate::state::{Counters, Group, Link}; -use crate::wire::{self, Address, Hello, Refusal}; +use crate::wire::{self, Address, Hello, Medium, Refusal}; use crate::PORT; /// The device under a link, as its driver drives it. pub trait Nic { + /// What the device's link is carried over. + const MEDIUM: Medium; + fn mac(&self) -> [u8; 6]; /// The claim, for the poller: readable means an interrupt has landed. @@ -131,12 +134,12 @@ struct Session<'a> { impl<'a> Session<'a> { /// A fresh region and its end, said hello with over `conn`. - fn open(label: &'a str, nic: &impl Nic, conn: Connection) -> Self { + fn open(label: &'a str, nic: &N, conn: Connection) -> Self { let region = Region::create().unwrap_or_else(|why| panic!("{label}: no region for a link: {why:?}")); let end = Driver::new(region.words()); let mac = Address::new(nic.mac()) .unwrap_or_else(|| panic!("{label}: the device's address is a group's, which no link carries")); - let hello = Hello { mac, label: label.into() }; + let hello = Hello { mac, medium: N::MEDIUM, label: label.into() }; let shared = region.share().unwrap_or_else(|why| panic!("{label}: the link's region would not share: {why:?}")); // A connection that will not take the hello is one whose stack has // gone: its hang-up is read in the first pass. diff --git a/lib/link/src/state.rs b/lib/link/src/state.rs index fb33c6c7b1f..ad3333850be 100644 --- a/lib/link/src/state.rs +++ b/lib/link/src/state.rs @@ -8,11 +8,15 @@ pub enum Link { /// The device reports no link at all, as virtio's network device without /// `VIRTIO_NET_F_STATUS` does; the stack takes it as up. Unreported, + /// The device reports its link up and no speed, as virtio's network + /// device with `VIRTIO_NET_F_STATUS` and without `VIRTIO_NET_F_SPEED_DUPLEX`. + UpWithoutSpeed, } const DOWN: u32 = 0; const UNREPORTED: u32 = 1; const UP: u32 = 2; +const UP_WITHOUT_SPEED: u32 = 3; const FULL_DUPLEX: u32 = 1 << 2; const MBPS_SHIFT: u32 = 8; @@ -27,6 +31,7 @@ impl Link { match self { Self::Down => DOWN, Self::Unreported => UNREPORTED, + Self::UpWithoutSpeed => UP_WITHOUT_SPEED, Self::Up { mbps, full_duplex } => { let mbps = mbps.clamp(1, u32::MAX >> MBPS_SHIFT); UP | if full_duplex { FULL_DUPLEX } else { 0 } | mbps << MBPS_SHIFT @@ -41,6 +46,7 @@ impl Link { match word & 0b11 { DOWN if word == DOWN => Some(Self::Down), UNREPORTED if word == UNREPORTED => Some(Self::Unreported), + UP_WITHOUT_SPEED if word == UP_WITHOUT_SPEED => Some(Self::UpWithoutSpeed), UP if reserved == 0 && mbps > 0 => Some(Self::Up { mbps, full_duplex: word & FULL_DUPLEX != 0 }), _ => None, } @@ -175,6 +181,7 @@ mod tests { for link in [ Link::Down, Link::Unreported, + Link::UpWithoutSpeed, Link::Up { mbps: 10, full_duplex: false }, Link::Up { mbps: 1000, full_duplex: true }, Link::Up { mbps: u32::MAX >> 8, full_duplex: true }, @@ -185,10 +192,10 @@ mod tests { } /// Every word no driver writes is refused: an up link of no speed, a - /// reserved bit, a down or unreported link carrying fields, a fourth tag. + /// reserved bit, a down, unreported or speedless link carrying fields. #[test] fn a_word_no_driver_writes_names_no_link() { - for word in [UP, UP | FULL_DUPLEX, UP | 1 << 3 | 1 << 8, DOWN | 1 << 8, UNREPORTED | FULL_DUPLEX, 3, 3 | 1 << 8, u32::MAX] { + for word in [UP, UP | FULL_DUPLEX, UP | 1 << 3 | 1 << 8, DOWN | 1 << 8, UNREPORTED | FULL_DUPLEX, UP_WITHOUT_SPEED | 1 << 8, UP_WITHOUT_SPEED | FULL_DUPLEX, u32::MAX] { assert_eq!(Link::decode(word), None, "{word:#x}"); } } diff --git a/lib/link/src/wire.rs b/lib/link/src/wire.rs index 7673a8a5591..fe3769dfae3 100644 --- a/lib/link/src/wire.rs +++ b/lib/link/src/wire.rs @@ -35,43 +35,74 @@ pub const MSG_REFUSED: u32 = 3; /// The longest driver name a hello carries. pub const MAX_LABEL: usize = 32; -/// Who a driver is: its device's address, which is an individual one, and its -/// name for the stack's lines: ASCII letters, digits and `-`. +/// What a link is carried over, which the stack ranks links by: wired before +/// wireless. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Medium { + Wired, + Wireless, +} + +impl Medium { + fn encode(self) -> u8 { + match self { + Self::Wired => 1, + Self::Wireless => 2, + } + } + + fn decode(byte: u8) -> Option { + match byte { + 1 => Some(Self::Wired), + 2 => Some(Self::Wireless), + _ => None, + } + } +} + +/// Who a driver is: its device's address, which is an individual one, what +/// the link is carried over, and its name for the stack's lines: ASCII +/// letters, digits and `-`. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Hello { pub mac: Address, + pub medium: Medium, pub label: String, } impl Hello { - pub const MAX_BYTES: usize = 6 + MAX_LABEL; + pub const MAX_BYTES: usize = 7 + MAX_LABEL; pub fn encode(&self) -> Vec { - let mut out = Vec::with_capacity(6 + self.label.len()); + let mut out = Vec::with_capacity(7 + self.label.len()); out.extend_from_slice(&self.mac.bytes()); + out.push(self.medium.encode()); out.extend_from_slice(self.label.as_bytes()); out } /// `None` for bytes [`Self::encode`] does not make: too short, a group's - /// address, or a name that is empty, too long or not one. + /// address, no medium, or a name that is empty, too long or not one. pub fn decode(bytes: &[u8]) -> Option { - let (mac, label) = bytes.split_first_chunk::<6>()?; + let (mac, rest) = bytes.split_first_chunk::<6>()?; let mac = Address::new(*mac)?; + let (&medium, label) = rest.split_first()?; + let medium = Medium::decode(medium)?; let named = |b: &u8| b.is_ascii_alphanumeric() || *b == b'-'; if label.is_empty() || label.len() > MAX_LABEL || !label.iter().all(named) { return None; } - Some(Self { mac, label: String::from(core::str::from_utf8(label).ok()?) }) + Some(Self { mac, medium, label: String::from(core::str::from_utf8(label).ok()?) }) } } /// Why the stack would not open a link. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Refusal { - /// The stack drives one link, and its address is another's, or a link of - /// this address is open and alive (`issues/netstack-drives-one-card.md`). - OneLink, + /// A link of this address is open and alive. + Open, + /// The stack drives the most links it drives. + Full, /// The stack ended this address's last link for what its driver wrote. /// The driver ends on it, so its supervisor's restart budget bounds a /// driver that keeps writing what no driver writes. @@ -83,18 +114,20 @@ pub enum Refusal { impl Refusal { pub fn encode(self) -> [u8; 4] { match self { - Self::OneLink => 1u32, + Self::Open => 1u32, Self::Malformed => 2, Self::Faulted => 3, + Self::Full => 4, } .to_le_bytes() } pub fn decode(bytes: &[u8]) -> Option { match u32::from_le_bytes(bytes.try_into().ok()?) { - 1 => Some(Self::OneLink), + 1 => Some(Self::Open), 2 => Some(Self::Malformed), 3 => Some(Self::Faulted), + 4 => Some(Self::Full), _ => None, } } @@ -110,9 +143,10 @@ mod tests { #[test] fn a_hello_reads_back_as_written() { - let hello = Hello { mac: mac([0x52, 0x54, 0, 0x12, 0x34, 0x56]), label: "virtio-net".into() }; + let hello = Hello { mac: mac([0x52, 0x54, 0, 0x12, 0x34, 0x56]), medium: Medium::Wired, label: "virtio-net".into() }; assert_eq!(Hello::decode(&hello.encode()), Some(hello)); - let longest = Hello { mac: mac([2; 6]), label: "a".repeat(MAX_LABEL) }; + let longest = Hello { mac: mac([2; 6]), medium: Medium::Wireless, label: "a".repeat(MAX_LABEL) }; + assert_eq!(longest.encode().len(), Hello::MAX_BYTES); assert_eq!(Hello::decode(&longest.encode()), Some(longest)); } @@ -120,8 +154,12 @@ mod tests { /// group's address among them: the node is made with the address. #[test] fn a_hello_no_driver_writes_is_refused() { - let named = |mac: [u8; 6], label: &[u8]| [mac.as_slice(), label].concat(); + let named = |mac: [u8; 6], label: &[u8]| [mac.as_slice(), &[1], label].concat(); assert_eq!(Hello::decode(&[2, 0, 0, 0, 0]), None, "short of an address"); + assert_eq!(Hello::decode(&[2, 0, 0, 0, 0, 0]), None, "no medium"); + for medium in [0, 3, 0xff] { + assert_eq!(Hello::decode(&[[2, 0, 0, 0, 0, 0, medium].as_slice(), b"i219"].concat()), None, "medium {medium}"); + } assert_eq!(Hello::decode(&named([0x01, 0x00, 0x5e, 0, 0, 0xfb], b"i219")), None, "a group's address"); assert_eq!(Hello::decode(&named([0xff; 6], b"i219")), None, "the broadcast address"); assert_eq!(Hello::decode(&named([2; 6], &[b'a'; MAX_LABEL + 1])), None, "a name too long"); @@ -133,7 +171,7 @@ mod tests { #[test] fn every_refusal_reads_back_and_no_other_word_does() { - for why in [Refusal::OneLink, Refusal::Malformed, Refusal::Faulted] { + for why in [Refusal::Open, Refusal::Full, Refusal::Malformed, Refusal::Faulted] { assert_eq!(Refusal::decode(&why.encode()), Some(why)); } assert_eq!(Refusal::decode(&0u32.to_le_bytes()), None); diff --git a/lib/virtio/src/pci.rs b/lib/virtio/src/pci.rs index 0cb974c7d25..57cf2081c7e 100644 --- a/lib/virtio/src/pci.rs +++ b/lib/virtio/src/pci.rs @@ -365,8 +365,15 @@ impl Wires { /// Where in the BAR the 32-bit device-specific field `at` is, read or /// written 32 bits wide as §4.1.3.1 has one. fn device32(&mut self, at: usize) -> Result { + self.device_field(at, 4) + } + + /// Where the `width`-byte field `at` bytes into the device-specific + /// structure is, refusing one that is not a whole aligned field of it + /// (§4.1.3.1). + fn device_field(&mut self, at: usize, width: usize) -> Result { let Region { at: base, bytes } = self.device; - if at.checked_add(4).is_none_or(|end| end > bytes) || !at.is_multiple_of(4) { + if at.checked_add(width).is_none_or(|end| end > bytes) || !at.is_multiple_of(width) { return Err(self.refuse(Refusal::PastDeviceConfig { at, bytes })); } Ok(base + at) @@ -576,6 +583,13 @@ impl Live { Ok(self.wires.regs.read32(at)) } + /// The 16-bit field `at` bytes into the device-specific structure, read + /// 16 bits wide (§4.1.3.1). A refusal sets `FAILED`. + pub fn device_read16(&mut self, at: usize) -> Result { + let at = self.wires.device_field(at, 2)?; + Ok(self.wires.regs.read16(at)) + } + /// Write the 32-bit field `at` bytes into the device-specific structure. pub fn device_write32(&mut self, at: usize, value: u32) -> Result<(), Refusal> { let at = self.wires.device32(at)?; diff --git a/lib/virtio/src/tests.rs b/lib/virtio/src/tests.rs index c19d0c0881c..c6628770724 100644 --- a/lib/virtio/src/tests.rs +++ b/lib/virtio/src/tests.rs @@ -482,6 +482,25 @@ fn a_live_device_reads_and_writes_its_own_fields_inside_the_structure() { } } +/// §4.1.3.1: a 16-bit field is read 16 bits wide, and one that is not a whole +/// aligned word of the structure is refused, with `FAILED`, and not read. +#[test] +fn a_live_devices_16_bit_field_is_read_16_bits_wide_and_only_inside_the_structure() { + let machine = Machine::new(); + let (mut device, _queue) = live(&machine); + machine.forget_trace(); + assert_eq!(device.device_read16(4), Ok(0x5634)); + assert_eq!(machine.trace(), [Event::DeviceConfig { at: 4 }]); + for at in [6, 5, usize::MAX] { + let machine = Machine::new(); + let (mut device, _queue) = live(&machine); + machine.forget_trace(); + assert_eq!(device.device_read16(at), Err(Refusal::PastDeviceConfig { at, bytes: 6 })); + let failed = status::ACKNOWLEDGE | status::DRIVER | status::FEATURES_OK | status::DRIVER_OK | status::FAILED; + assert_eq!(machine.trace(), [Event::Status(failed)]); + } +} + // --- PCI 3.0 §6.7: the capability list --- /// A function's configuration space: 256 bytes, and the offsets a claim diff --git a/sdk/libc/src/errcode.rs b/sdk/libc/src/errcode.rs index 2eaa6163f7e..d4ad92a6e2d 100644 --- a/sdk/libc/src/errcode.rs +++ b/sdk/libc/src/errcode.rs @@ -52,6 +52,7 @@ pub(crate) const ENOTCONN: i32 = 107; pub(crate) const ETIMEDOUT: i32 = 110; pub(crate) const ECONNREFUSED: i32 = 111; pub(crate) const EHOSTUNREACH: i32 = 113; +pub(crate) const EADDRNOTAVAIL: i32 = 99; pub(crate) const ESTALE: i32 = 116; /// The errno a kernel refusal is answered with. @@ -111,6 +112,7 @@ pub(crate) fn of_stream(refused: toyos::net::stream::Refused) -> i32 { Refused::TimedOut | Refused::ConnectionTimedOut => ETIMEDOUT, Refused::Reset => ECONNRESET, Refused::Unreachable => EHOSTUNREACH, + Refused::AddressGone => EADDRNOTAVAIL, Refused::Aborted => ECONNABORTED, Refused::Shut => EPIPE, } diff --git a/sdk/std/sys/net/connection.rs b/sdk/std/sys/net/connection.rs index 680e1475680..3bef6d9d3de 100644 --- a/sdk/std/sys/net/connection.rs +++ b/sdk/std/sys/net/connection.rs @@ -93,6 +93,7 @@ pub fn stream_refused(refusal: Refused) -> io::Error { Refused::Reset => io::const_error!(io::ErrorKind::ConnectionReset, "connection reset"), Refused::ConnectionTimedOut => io::const_error!(io::ErrorKind::TimedOut, "the connection timed out"), Refused::Unreachable => io::const_error!(io::ErrorKind::HostUnreachable, "the peer is unreachable"), + Refused::AddressGone => io::const_error!(io::ErrorKind::AddrNotAvailable, "this machine gave up the connection's address"), Refused::Aborted => io::const_error!(io::ErrorKind::ConnectionAborted, "netstack ended the stream"), Refused::Shut => SHUT_DOWN, } diff --git a/sdk/toyos/src/files.rs b/sdk/toyos/src/files.rs index d2d44b1b6fc..6aabdd02d30 100644 --- a/sdk/toyos/src/files.rs +++ b/sdk/toyos/src/files.rs @@ -537,7 +537,7 @@ mod tests { every.extend((0..=255u8).filter_map(fs::Error::from_byte).map(Error::Served)); let streams: alloc::vec::Vec = (0..=255u8).filter_map(crate::net::stream::Refused::from_byte).map(Error::Stream).collect(); - assert_eq!(streams.len(), 7, "every stream refusal but a foreign handle's"); + assert_eq!(streams.len(), 8, "every stream refusal but a foreign handle's"); every.extend(streams); assert!(every.len() > 40, "only {} errors", every.len()); for e in every { diff --git a/sdk/toyos/src/net/stream.rs b/sdk/toyos/src/net/stream.rs index 94db3a7b9b8..454b391e415 100644 --- a/sdk/toyos/src/net/stream.rs +++ b/sdk/toyos/src/net/stream.rs @@ -4,7 +4,8 @@ //! with the connect or the accept. [`LAYOUT`] is its one declaration: a header //! of [`WORDS`] words, each on a 64-byte line of its own, then the receive ring //! netstack fills and the send ring the client fills. The receive ring's `end` -//! is the peer's FIN, [`RESET`], [`TIMED_OUT`] or [`UNREACHABLE`]; the send +//! is the peer's FIN, [`RESET`], [`TIMED_OUT`], [`UNREACHABLE`] or +//! [`ADDRESS_GONE`]; the send //! ring's is the client's shutdown; [`NODELAY`] is the client's Nagle switch //! (RFC 9293 §3.7.4). The rest of the header is the session's: the gates, the //! receiving half's shutdown and the hang-up any holder saw. @@ -111,6 +112,9 @@ pub const RESET: u32 = 2; pub const TIMED_OUT: u32 = 3; /// The receive ring's `end`: an ICMP error ended the connection. pub const UNREACHABLE: u32 = 4; +/// The receive ring's `end`: this machine gave up the connection's local +/// address, its link's lease ended or another host took it. +pub const ADDRESS_GONE: u32 = 5; // ---- a holder's connection ---- @@ -239,6 +243,8 @@ pub enum Refused { ConnectionTimedOut, /// An ICMP error ended the connection. Unreachable, + /// This machine gave up the connection's local address. + AddressGone, /// netstack went away, or wrote what no netstack writes. Aborted, /// The sending half was shut down. @@ -259,6 +265,7 @@ impl Refused { Self::Unreachable => 4, Self::Aborted => 5, Self::Shut => 6, + Self::AddressGone => 7, }) } @@ -271,6 +278,7 @@ impl Refused { 4 => Self::Unreachable, 5 => Self::Aborted, 6 => Self::Shut, + 7 => Self::AddressGone, _ => return None, }) } @@ -283,6 +291,7 @@ impl From for Refused { Answer::TimedOut => Self::TimedOut, Answer::Failed(TIMED_OUT) => Self::ConnectionTimedOut, Answer::Failed(UNREACHABLE) => Self::Unreachable, + Answer::Failed(ADDRESS_GONE) => Self::AddressGone, Answer::Failed(_) => Self::Reset, Answer::Shut => Self::Shut, Answer::Gone | Answer::Violation(_) => Self::Aborted, diff --git a/system/i219/src/nic.rs b/system/i219/src/nic.rs index 1f724d64346..9910227fab2 100644 --- a/system/i219/src/nic.rs +++ b/system/i219/src/nic.rs @@ -240,6 +240,8 @@ impl Nic { } impl toyos_link::driver::Nic for Nic { + const MEDIUM: toyos_link::wire::Medium = toyos_link::wire::Medium::Wired; + fn mac(&self) -> [u8; 6] { self.mac } diff --git a/system/netstack/dhcp/src/lib.rs b/system/netstack/dhcp/src/lib.rs index f3790c69f61..70d17fd55e9 100644 --- a/system/netstack/dhcp/src/lib.rs +++ b/system/netstack/dhcp/src/lib.rs @@ -38,6 +38,7 @@ use toyos_net_wire::addr::{is_host, Cidr}; use toyos_net_wire::ethernet::{IndividualMac, MacAddr}; use toyos_net_wire::Instant; +pub use message::addressed_to; use message::{Build, Kind, Reply, ReplyKind}; toyos_net_wire::counters! { diff --git a/system/netstack/dhcp/src/message.rs b/system/netstack/dhcp/src/message.rs index 5ed8b6f8311..11e90557c27 100644 --- a/system/netstack/dhcp/src/message.rs +++ b/system/netstack/dhcp/src/message.rs @@ -153,6 +153,16 @@ fn addresses(bytes: &[u8]) -> Vec { bytes.as_chunks::<4>().0.iter().map(|&a| Ipv4Addr::from(a)).collect() } +/// The client hardware address a server's reply names (RFC 2131 §2), where it is a reply over +/// Ethernet: which of a host's links' clients it is for. Nothing else of it is read. +pub fn addressed_to(bytes: &[u8]) -> Option { + let (&[op, htype, hlen, ..], _) = bytes.split_first_chunk::<4>()?; + if (op, htype, hlen) != (2, 1, 6) { + return None; + } + bytes.get(28..34)?.first_chunk::<6>().map(|&mac| MacAddr(mac)) +} + impl Reply { pub fn parse(bytes: &[u8], mac: MacAddr) -> Result { let (fixed, options) = bytes.split_at_checked(OPTIONS_AT).ok_or(Counter::Truncated)?; diff --git a/system/netstack/dhcp/tests/common/mod.rs b/system/netstack/dhcp/tests/common/mod.rs index 2821031cd3c..7e5a00da060 100644 --- a/system/netstack/dhcp/tests/common/mod.rs +++ b/system/netstack/dhcp/tests/common/mod.rs @@ -248,7 +248,7 @@ pub fn ms(i: Instant) -> u64 { pub fn framed(t: &Transmission) -> Vec { use toyos_net_ip::{Ip, Sent, FRAME}; let mut ip = Ip::new(Instant::from_nanos(0), [0x5a; 16]); - let if0 = ip.add_interface(Instant::from_nanos(0), IndividualMac::new(MAC_A).unwrap()); + let if0 = ip.add_interface(Instant::from_nanos(0), IndividualMac::new(MAC_A).unwrap(), toyos_net_ip::Rank(0)); ip.link_up(Instant::from_nanos(0), if0).unwrap(); ip.add_address(Instant::from_nanos(0), if0, A, 24).unwrap(); let setup = |ip: &mut Ip, until: Instant| { @@ -274,12 +274,12 @@ pub fn framed(t: &Transmission) -> Vec { let socket = udp.bind(&ip, Ipv4Addr::UNSPECIFIED, toyos_net_wire::Port::new(68), || 0).unwrap(); udp.set_acquisition(socket).unwrap(); udp.set_broadcast(socket, true).unwrap(); - udp.send_from(&mut ip, socket, t.source, t.destination, 67, &t.payload).unwrap(); + udp.send_from(&mut ip, socket, toyos_net_udp::Origin { iface: if0, source: t.source }, t.destination, 67, &t.payload).unwrap(); let mut frame = Vec::new(); let sent = udp.serve(toyos_net_udp::Sender::Socket(socket), |out| { let mut buf = [0u8; FRAME]; - match ip.send_udp(now, out, &mut buf) { - Ok(Sent::Frame(n)) => frame = buf[..n].to_vec(), + match ip.send_udp(now, &usize::MAX, out, &mut buf) { + Ok(Sent::Frame { len: n, .. }) => frame = buf[..n].to_vec(), other => panic!("{other:?}"), } toyos_net_udp::Offer::Taken diff --git a/system/netstack/dhcp/tests/parse.rs b/system/netstack/dhcp/tests/parse.rs index 7bb962af808..75f6ec492ea 100644 --- a/system/netstack/dhcp/tests/parse.rs +++ b/system/netstack/dhcp/tests/parse.rs @@ -318,3 +318,18 @@ fn s_dhcp_dh_063_a_refused_offer_then_another() { let payload = &sent(&out).payload; assert_eq!(payload[243..249], [54, 4, 192, 0, 2, 253]); } + +/// A reply names the client it is for, which a host with several links reads to tell which +/// link's client is to have it; a message that is no Ethernet reply names none. +#[test] +fn a_reply_names_its_client_and_nothing_else_does() { + let reply = offer(); + assert_eq!(toyos_dhcp::addressed_to(&reply), Some(MAC_A)); + let mut request = reply.clone(); + request[0] = 1; + assert_eq!(toyos_dhcp::addressed_to(&request), None, "a request"); + let mut token_ring = reply.clone(); + token_ring[1] = 6; + assert_eq!(toyos_dhcp::addressed_to(&token_ring), None, "another hardware type"); + assert_eq!(toyos_dhcp::addressed_to(&reply[..33]), None, "short of its chaddr"); +} diff --git a/system/netstack/dhcp/tests/stack.rs b/system/netstack/dhcp/tests/stack.rs index f49916b1ac7..6025611461b 100644 --- a/system/netstack/dhcp/tests/stack.rs +++ b/system/netstack/dhcp/tests/stack.rs @@ -57,7 +57,7 @@ impl Stack { /// A's interface up with no address, netstack's acquisition socket on port 68. fn new() -> Self { let mut ip = Ip::new(Instant::from_nanos(0), [0x5a; 16]); - let if0 = ip.add_interface(Instant::from_nanos(0), IndividualMac::new(MAC_A).unwrap()); + let if0 = ip.add_interface(Instant::from_nanos(0), IndividualMac::new(MAC_A).unwrap(), toyos_net_ip::Rank(0)); ip.link_up(Instant::from_nanos(0), if0).unwrap(); let mut udp = Udp::new(); let socket = udp.bind(&ip, Ipv4Addr::UNSPECIFIED, Port::new(68), || 0).unwrap(); diff --git a/system/netstack/ip/src/config.rs b/system/netstack/ip/src/config.rs index bf7360d1dca..bc0f25138e3 100644 --- a/system/netstack/ip/src/config.rs +++ b/system/netstack/ip/src/config.rs @@ -117,11 +117,11 @@ impl Ip { self.ifaces.get(iface.0).map(|i| i.igmp.mode()) } - /// A transport's advice about its peer `remote`; it lands on the next hop's entry, the - /// gateway's for an off-link peer. - pub fn advise(&mut self, now: Instant, remote: Ipv4Addr, advice: Advice) { + /// A transport's advice about its peer `remote`, whose flow is from `source`; it lands on + /// the next hop's entry on the flow's own link, the gateway's for an off-link peer. + pub fn advise(&mut self, now: Instant, remote: Ipv4Addr, source: Source, advice: Advice) { let now = self.clock(now); - let Ok(route) = route::lookup(&self.ifaces, remote, Source::Any, None) else { return }; + let Ok(route) = route::lookup(&self.ifaces, &self.order, remote, source, None) else { return }; let NextHop::Neighbour(next_hop) = route.next_hop else { return }; let Some((i, mut cx)) = self.split(now, route.iface) else { return }; nud::advise(i, &mut cx, next_hop, advice == Advice::Confirmed); diff --git a/system/netstack/ip/src/egress.rs b/system/netstack/ip/src/egress.rs index dd91fe40f37..3476396178f 100644 --- a/system/netstack/ip/src/egress.rs +++ b/system/netstack/ip/src/egress.rs @@ -1,10 +1,11 @@ //! Egress. [ip]'s own frames wait in one FIFO the transmit opportunity drains ahead of the //! data flows: an ARP or IGMP frame is built when it leaves, from the state of that moment, and -//! the timer it feeds starts then. The FIFO holds at most CONTROL_QUEUE frames, at most -//! ECHO_REPLIES of them echo replies; one of [ip]'s own frames past either is dropped and -//! counted, and its producer moves on as if it had left. Three kinds are never dropped: a -//! released datagram's turn, an ACD probe and an ACD announcement. One that finds the FIFO full -//! waits behind it and enters as room appears, ahead of any frame queued later (RFC 5227 +//! the timer it feeds starts then. The FIFO holds at most CONTROL_QUEUE frames of each +//! interface, at most ECHO_REPLIES of them echo replies, so a link that takes nothing holds no +//! other link's frames back; one of [ip]'s own frames past either is dropped and counted, and +//! its producer moves on as if it had left. Three kinds are never dropped: a released datagram's +//! turn, an ACD probe and an ACD announcement. One that finds its interface's share full waits +//! and enters as room appears there, ahead of any frame of that interface queued later (RFC 5227 //! §2.1). A turn holds no datagram: it names the entry whose queue does, and goes with that entry. //! //! Only [ip]'s own ICMP messages wait here for a next hop. A transport's datagram is framed into @@ -76,61 +77,67 @@ pub(crate) struct Turn { #[derive(Debug, Default)] pub(crate) struct Control { items: VecDeque, - echoes: usize, - /// What may not be dropped and found `items` full, in order: empty whenever it has room. + /// What may not be dropped and found its interface's share full, in order: none of an + /// interface whose share has room. waiting: VecDeque, } +fn is_echo(item: &Item) -> bool { + matches!(item, Item::Frame { kind: FrameKind::Echo, .. }) +} + impl Control { - fn full(&self) -> bool { - self.items.len() >= CONTROL_QUEUE + fn queued(&self, iface: IfIndex) -> usize { + self.items.iter().filter(|item| item.iface() == iface).count() + } + + fn full(&self, iface: IfIndex) -> bool { + self.queued(iface) >= CONTROL_QUEUE } /// Queues one of [ip]'s own frames; `false` when it was dropped and counted. pub fn push(&mut self, item: Item, log: &mut Log) -> bool { - let echo = matches!(item, Item::Frame { kind: FrameKind::Echo, .. }); - if echo && self.echoes >= ECHO_REPLIES { + let iface = item.iface(); + if is_echo(&item) && self.items.iter().filter(|i| i.iface() == iface && is_echo(i)).count() >= ECHO_REPLIES { log.count(Counter::IcmpEchoReplyDropped); return false; } - if self.full() { + if self.full(iface) { log.count(Counter::IpControlQueueFull); return false; } - if echo { - self.echoes = self.echoes.saturating_add(1); - } self.items.push_back(item); true } /// Queues a turn, a probe or an announcement, which waits for room rather than being dropped. pub fn hold(&mut self, item: Item) { - if self.full() { + if self.full(item.iface()) { self.waiting.push_back(item); } else { self.items.push_back(item); } } - fn pop(&mut self) -> Option { - let item = self.items.pop_front()?; - self.forget(&item); + /// The oldest item whose interface `room` says has room. + fn pop(&mut self, mut room: impl FnMut(IfIndex) -> bool) -> Option { + let at = self.items.iter().position(|item| room(item.iface()))?; + let item = self.items.remove(at)?; self.refill(); Some(item) } - fn forget(&mut self, item: &Item) { - if matches!(item, Item::Frame { kind: FrameKind::Echo, .. }) { - self.echoes = self.echoes.saturating_sub(1); - } - } - - /// Room goes to the oldest item waiting for it. + /// Room goes to the oldest item waiting for its interface's. fn refill(&mut self) { - while !self.full() { - let Some(item) = self.waiting.pop_front() else { return }; - self.items.push_back(item); + let mut at = 0; + while let Some(item) = self.waiting.get(at) { + if self.full(item.iface()) { + at = at.saturating_add(1); + continue; + } + if let Some(item) = self.waiting.remove(at) { + self.items.push_back(item); + } } } @@ -154,16 +161,39 @@ impl Control { /// Drops everything waiting for `iface`, whose link went down. pub fn purge(&mut self, iface: IfIndex) { - let (gone, kept): (Vec, Vec) = self.items.drain(..).partition(|item| item.iface() == iface); - self.items = kept.into(); - for item in &gone { - self.forget(item); - } + self.items.retain(|item| item.iface() != iface); self.waiting.retain(|item| item.iface() != iface); self.refill(); } } +/// The room a transmit opportunity has, interface by interface. A count is +/// room on every interface alike. +pub trait Credit { + /// Whether `iface` takes one frame more. + fn room(&self, iface: IfIndex) -> bool; + /// One frame left by `iface`. + fn spend(&mut self, iface: IfIndex); +} + +impl Credit for usize { + fn room(&self, _: IfIndex) -> bool { + *self > 0 + } + fn spend(&mut self, _: IfIndex) { + *self = self.saturating_sub(1); + } +} + +impl Credit for &mut C { + fn room(&self, iface: IfIndex) -> bool { + (**self).room(iface) + } + fn spend(&mut self, iface: IfIndex) { + (**self).spend(iface); + } +} + /// A UDP datagram a transport hands [ip] at a transmit opportunity. #[derive(Clone, Copy, Debug)] pub struct UdpOut<'a> { @@ -174,16 +204,23 @@ pub struct UdpOut<'a> { /// Whether its socket held the broadcast permission when it accepted the datagram: one /// without it never leaves in a link broadcast, whatever the prefixes have become since. pub broadcast: bool, + /// The interface it may leave by, where its send named one. + pub iface: Option, pub datagram: UdpBuilder<'a>, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Sent { - /// This many bytes of the caller's buffer are the frame: one credit spent. - Frame(usize), - /// Not taken: this next hop's link address is being asked for. [`Event::Resolved`] or - /// [`Event::Failed`] names it when that ends, unless the routes change first. - Pending(Ipv4Addr), + /// This many bytes of the caller's buffer are the frame, for this interface: one of its + /// credit to spend. + Frame { len: usize, iface: IfIndex }, + /// Not taken: the link address of this next hop on `iface` is being asked for. + /// [`Event::Resolved`] or [`Event::Failed`] names it when that ends, unless the routes change + /// first. + Pending { iface: IfIndex, next_hop: Ipv4Addr }, + /// Not taken: its interface has no room in this opportunity. The hop is its next hop, or + /// its destination where it has none on the link. + NoRoom { iface: IfIndex, hop: Ipv4Addr }, } /// Where a datagram goes on the link. @@ -205,15 +242,17 @@ fn build_vec(source: IndividualMac, destination: MacAddr, body: &B } impl Ip { - /// A transmit opportunity with room for `credit` frames: [ip]'s own frames and the datagrams + /// A transmit opportunity with `credit`'s room: [ip]'s own frames and the datagrams /// resolution released, in the order they were queued, each handed to `sink` with its - /// interface. Returns how many left. - pub fn transmit(&mut self, now: Instant, credit: usize, mut sink: impl FnMut(IfIndex, &[u8])) -> usize { + /// interface; one whose interface has no room waits without holding back another's. Returns + /// how many left. + pub fn transmit(&mut self, now: Instant, mut credit: impl Credit, mut sink: impl FnMut(IfIndex, &[u8])) -> usize { let now = self.clock(now); let mut sent = 0usize; - while sent < credit { - let Some(item) = self.control.pop() else { break }; + while let Some(item) = self.control.pop(|iface| credit.room(iface)) { + let iface = item.iface(); if self.emit(now, item, &mut sink) { + credit.spend(iface); sent = sent.saturating_add(1); } } @@ -301,7 +340,7 @@ impl Ip { /// is known, and otherwise not taken, its next hop asked for. The route is the one of this /// moment, so a link broadcast is refused a datagram that does not carry the permission. A /// refusal is counted and the datagram's flow is told it is unreachable. - pub fn send_udp(&mut self, now: Instant, out: &UdpOut<'_>, frame: &mut [u8; FRAME]) -> Result { + pub fn send_udp(&mut self, now: Instant, credit: &impl Credit, out: &UdpOut<'_>, frame: &mut [u8; FRAME]) -> Result { let now = self.clock(now); let flow = Flow { source: out.source, @@ -310,7 +349,14 @@ impl Ip { destination_port: out.datagram.destination, }; let source = if out.source.is_unspecified() { Source::Unspecified } else { Source::Bound(out.source) }; - let sent = self.route(out.destination, source, None).and_then(|route| { + let sent = self.route(out.destination, source, out.iface).and_then(|route| { + if !credit.room(route.iface) { + let hop = match route.next_hop { + NextHop::Neighbour(next_hop) => next_hop, + NextHop::Broadcast | NextHop::Multicast(_) => out.destination, + }; + return Ok(Sent::NoRoom { iface: route.iface, hop }); + } let builder = Ipv4Builder { source: Ipv4Source::new(route.source).map_err(|_| Counter::RouteNoSourceAddress)?, destination: out.destination, @@ -322,8 +368,8 @@ impl Ip { self.fits(&builder)?; let Some((i, mut cx)) = self.split(now, route.iface) else { return Err(Counter::UnknownInterface) }; match hop(i, &mut cx, &route, out.destination, out.broadcast)? { - Hop::To(mac) => build(i.mac, mac, &builder, frame).map(Sent::Frame).ok_or(Counter::IpExceedsMtu), - Hop::Asked(next_hop) => Ok(Sent::Pending(next_hop)), + Hop::To(mac) => build(i.mac, mac, &builder, frame).map(|len| Sent::Frame { len, iface: route.iface }).ok_or(Counter::IpExceedsMtu), + Hop::Asked(next_hop) => Ok(Sent::Pending { iface: route.iface, next_hop }), } }); if sent.is_err() { diff --git a/system/netstack/ip/src/iface.rs b/system/netstack/ip/src/iface.rs index 3d3525557f2..43a235ded14 100644 --- a/system/netstack/ip/src/iface.rs +++ b/system/netstack/ip/src/iface.rs @@ -17,6 +17,7 @@ use crate::{acd, igmp, limits, nud, IfIndex, Ip}; pub(crate) struct Interface { pub mac: IndividualMac, + pub rank: crate::Rank, pub up: bool, /// In the order they were added. pub addresses: Vec
, diff --git a/system/netstack/ip/src/lib.rs b/system/netstack/ip/src/lib.rs index dad217a5965..4f0665505e5 100644 --- a/system/netstack/ip/src/lib.rs +++ b/system/netstack/ip/src/lib.rs @@ -68,7 +68,7 @@ use toyos_net_wire::Port; pub use addr::AddrState; pub use counters::{Counter, Counters, RefusalLog}; -pub use egress::{Sent, UdpOut, FRAME}; +pub use egress::{Credit, Sent, UdpOut, FRAME}; pub use igmp::IgmpMode; pub use limiter::Limiter; pub use nud::{Delaying, Held, Incomplete, Nud, Pending, Probing, Reachable, Released, Stale, Unreachable}; @@ -80,6 +80,11 @@ pub use toyos_net_wire::Instant; #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct IfIndex(usize); +/// Where an interface stands among the others for every choice between them: +/// the lower first, and among equals the one added first. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Rank(pub u8); + /// The IP MTU of every interface: an Ethernet body. pub const MTU: usize = toyos_net_wire::ethernet::MAX_BODY; @@ -315,6 +320,8 @@ pub enum Resolution { pub struct Ip { draws: draw::Draws, ifaces: Vec, + /// Every interface's index, by rank and then by when it was added. + order: Vec, timers: timers::Timers, control: egress::Control, limiter: Limiter, @@ -330,6 +337,7 @@ impl Ip { Self { draws, ifaces: Vec::new(), + order: Vec::new(), timers: timers::Timers::default(), control: egress::Control::default(), limiter, @@ -339,11 +347,15 @@ impl Ip { } } - pub fn add_interface(&mut self, now: Instant, mac: IndividualMac) -> IfIndex { + pub fn add_interface(&mut self, now: Instant, mac: IndividualMac, rank: Rank) -> IfIndex { let now = self.clock(now); let reachable = self.draws.reachable_time(); + let index = self.ifaces.len(); + let at = self.order.iter().position(|&n| self.ifaces.get(n).is_some_and(|i| i.rank > rank)).unwrap_or(self.order.len()); + self.order.insert(at, index); self.ifaces.push(iface::Interface { mac, + rank, up: false, addresses: Vec::new(), gateways: Vec::new(), @@ -356,7 +368,12 @@ impl Ip { acd: acd::Conflicts::default(), igmp: igmp::Igmp::default(), }); - IfIndex(self.ifaces.len().saturating_sub(1)) + IfIndex(index) + } + + /// The link address of `iface`. + pub fn mac(&self, iface: IfIndex) -> Option { + self.ifaces.get(iface.0).map(|i| i.mac) } pub fn counters(&self) -> &Counters { diff --git a/system/netstack/ip/src/route.rs b/system/netstack/ip/src/route.rs index fc05654905b..122758a6895 100644 --- a/system/netstack/ip/src/route.rs +++ b/system/netstack/ip/src/route.rs @@ -2,7 +2,8 @@ //! on-link gateways per interface; longest prefix first, then 169.254/16 on the link itself (RFC //! 3927 §2.6.2), then the active gateway, and no forwarding. A lookup considers only interfaces //! that are up, and with a bound source only the interface holding it (RFC 1122 §3.3.4.2, the -//! strong model). +//! strong model). Between interfaces that tie, the one of lower [`crate::Rank`] wins, and among +//! equal ranks the one added first. use core::net::Ipv4Addr; @@ -40,7 +41,8 @@ pub enum Source { Unspecified, } -pub(crate) fn lookup(ifaces: &[Interface], destination: Ipv4Addr, source: Source, bound: Option) -> Result { +/// `order` is every interface's index, by rank and then arrival. +pub(crate) fn lookup(ifaces: &[Interface], order: &[usize], destination: Ipv4Addr, source: Source, bound: Option) -> Result { let eligible = |index: usize, i: &Interface| { i.up && bound.is_none_or(|b| b.0 == index) @@ -49,7 +51,7 @@ pub(crate) fn lookup(ifaces: &[Interface], destination: Ipv4Addr, source: Source Source::Any | Source::Unspecified => true, } }; - let candidates = || ifaces.iter().enumerate().filter(|&(n, i)| eligible(n, i)); + let candidates = || order.iter().filter_map(|&n| ifaces.get(n).map(|i| (n, i))).filter(|&(n, i)| eligible(n, i)); let pick_source = |i: &Interface, toward: Ipv4Addr| match source { Source::Bound(addr) => Ok(addr), Source::Unspecified => Err(Counter::RouteNoSourceAddress), @@ -141,7 +143,7 @@ pub(crate) fn withdraw_off_link(i: &mut Interface, cx: &mut Cx<'_>) { impl Ip { /// Where a datagram to `destination` goes, or the refusal a transport reports. pub fn route(&mut self, destination: Ipv4Addr, source: Source, iface: Option) -> Result { - lookup(&self.ifaces, destination, source, iface).inspect_err(|&refusal| self.log.count(refusal)) + lookup(&self.ifaces, &self.order, destination, source, iface).inspect_err(|&refusal| self.log.count(refusal)) } /// Installs the interface's gateways, in order of preference (RFC 2132 §3.5). A list with any diff --git a/system/netstack/ip/tests/acd.rs b/system/netstack/ip/tests/acd.rs index b6932206e50..3c6f810bcfe 100644 --- a/system/netstack/ip/tests/acd.rs +++ b/system/netstack/ip/tests/acd.rs @@ -7,7 +7,7 @@ use std::time::Duration; use common::*; use toyos_net_ip::limits::acd::{ANNOUNCE_WAIT, PROBE_MAX, PROBE_MIN, PROBE_WAIT}; -use toyos_net_ip::{limits, AddrState, Counter, Event, IfIndex, Instant, Peer, Source}; +use toyos_net_ip::{limits, AddrState, Counter, Event, Instant, Peer, Source}; use toyos_net_wire::ethernet::MacAddr; fn ip4(a: u8, b: u8, c: u8, d: u8) -> Ipv4Addr { @@ -268,20 +268,29 @@ fn s_ip_acd_017_detection_starts_at_the_first_announcement() { assert_eq!(h.count(Counter::AcdDefended), 1); } -/// if1 at 198.51.100.1/24 beside if0, which holds no address. -fn beside_if1() -> (H, IfIndex) { +/// The address on if0 whose ARP replies fill its share of the control queue. +const ANSWERING: Ipv4Addr = Ipv4Addr::new(198, 51, 100, 1); + +/// if0 holding [`ANSWERING`] and nothing else yet: its share of the control queue, which is +/// its own, for the tests to fill with answers to it. +fn answering() -> H { let mut h = H::raw(); - let if1 = h.add_if1(); - h.assign(if1, ip4(198, 51, 100, 1), 24, 0); + let if0 = h.if0; + h.assign(if0, ANSWERING, 24, 0); h.settle(); - (h, if1) + h +} + +/// An ARP reply's frame: EtherType 0x0806, operation 2. +fn is_arp_reply(frame: &[u8]) -> bool { + frame.get(12..14) == Some(&[0x08, 0x06]) && frame.get(20..22) == Some(&[0, 2]) } #[test] fn s_ip_acd_018_a_full_control_queue_holds_probes_and_announcements() { - let (mut h, if1) = beside_if1(); + let mut h = answering(); let t0 = h.clock(); - h.fill_control_queue(if1, t0, ip4(198, 51, 100, 1)); + h.fill_control_queue(h.if0, t0, ANSWERING); h.ip.add_address(t0, h.if0, A, 24).unwrap(); let at = t0.after(Duration::from_secs(1)); while let Some(d) = h.ip.next_deadline().filter(|d| *d <= at) { @@ -295,7 +304,7 @@ fn s_ip_acd_018_a_full_control_queue_holds_probes_and_announcements() { let mut out = Vec::new(); h.ip.transmit(at, limits::CONTROL_QUEUE, |iface, f| out.push((iface, f.to_vec()))); assert_eq!(out.len(), limits::CONTROL_QUEUE); - assert!(out.iter().all(|(iface, _)| *iface == if1), "the replies queued first"); + assert!(out.iter().all(|(iface, f)| *iface == h.if0 && is_arp_reply(f)), "the replies queued first"); out.clear(); h.ip.transmit(at, 1, |iface, f| out.push((iface, f.to_vec()))); assert_eq!(out, [(h.if0, hex(V_ARP_PROBE))]); @@ -304,12 +313,13 @@ fn s_ip_acd_018_a_full_control_queue_holds_probes_and_announcements() { assert!(h.events.contains(&Event::Verified { iface: h.if0, addr: A })); assert_eq!(h.ip.address(h.if0, A), Some(AddrState::Assigned)); - // A probes with credit until its third probe leaves; then if1 fills the queue, and the first - // announcement it owes waits there, counted toward nothing until it leaves. - let (mut h, if1) = beside_if1(); + // A probes with credit until its third probe leaves; then answers to ANSWERING fill if0's + // share of the queue, and the first announcement A owes waits there, counted toward nothing + // until it leaves. + let mut h = answering(); h.ip.add_address(h.clock(), h.if0, A, 24).unwrap(); let p3 = third_probe(&mut h); - h.fill_control_queue(if1, p3, ip4(198, 51, 100, 1)); + h.fill_control_queue(h.if0, p3, ANSWERING); let verified = p3.after(ANNOUNCE_WAIT); h.ip.fire(verified); h.collect(); @@ -324,7 +334,7 @@ fn s_ip_acd_018_a_full_control_queue_holds_probes_and_announcements() { let mut out = Vec::new(); h.ip.transmit(t1, usize::MAX, |iface, f| out.push((iface, f.to_vec()))); assert_eq!(out.len(), limits::CONTROL_QUEUE + 1, "nothing more was queued"); - assert!(out[..limits::CONTROL_QUEUE].iter().all(|(iface, _)| *iface == if1), "the replies queued first"); + assert!(out[..limits::CONTROL_QUEUE].iter().all(|(iface, f)| *iface == h.if0 && is_arp_reply(f)), "the replies queued first"); assert_eq!(out[limits::CONTROL_QUEUE], (h.if0, hex(V_ARP_ANNOUNCE))); assert!(run_exact(&mut h, t1.after(Duration::from_millis(1_999))).is_empty()); assert_eq!(h.ip.address(h.if0, A), Some(AddrState::Announcing), "ANNOUNCE_INTERVAL runs from the hand-off"); diff --git a/system/netstack/ip/tests/clk.rs b/system/netstack/ip/tests/clk.rs index 3be58312317..628a7823384 100644 --- a/system/netstack/ip/tests/clk.rs +++ b/system/netstack/ip/tests/clk.rs @@ -3,7 +3,7 @@ mod common; use common::*; -use toyos_net_ip::{limits, AddrState, Counter, Instant, Nud}; +use toyos_net_ip::{limits, AddrState, Counter, Instant, Nud, Rank}; fn incomplete_b() -> H { let mut h = H::fixture_i(); @@ -87,7 +87,7 @@ type Trace = (Vec<(u64, Vec)>, Vec, Vec, std::time::Duration); fn trace(secret: [u8; 16], seed: u64) -> Trace { let mut ip = toyos_net_ip::Ip::new(Instant::from_nanos(0), secret); - let if0 = ip.add_interface(Instant::from_nanos(0), mac(MAC_A)); + let if0 = ip.add_interface(Instant::from_nanos(0), mac(MAC_A), Rank(0)); ip.link_up(Instant::from_nanos(0), if0).unwrap(); let mut rng = Rng(seed); let mut frames = Vec::new(); @@ -118,8 +118,8 @@ fn trace(secret: [u8; 16], seed: u64) -> Trace { destination: toyos_net_wire::Port::new(5001).unwrap(), data: b"x", }; - let out = toyos_net_ip::UdpOut { source: A, destination, ttl: toyos_net_wire::ipv4::Ttl::DEFAULT, broadcast: false, datagram }; - if let Ok(toyos_net_ip::Sent::Frame(n)) = ip.send_udp(at, &out, &mut frame) { + let out = toyos_net_ip::UdpOut { source: A, destination, ttl: toyos_net_wire::ipv4::Ttl::DEFAULT, broadcast: false, iface: None, datagram }; + if let Ok(toyos_net_ip::Sent::Frame { len: n, .. }) = ip.send_udp(at, &usize::MAX, &out, &mut frame) { frames.push((now, frame[..n].to_vec())); } } @@ -223,8 +223,8 @@ fn s_ip_clk_010_a_deadline_saturates() { destination: toyos_net_wire::Port::new(5001).unwrap(), data: b"hi", }; - let out = toyos_net_ip::UdpOut { source: A, destination: B, ttl: toyos_net_wire::ipv4::Ttl::DEFAULT, broadcast: false, datagram }; - assert_eq!(h.ip.send_udp(late, &out, &mut frame), Ok(toyos_net_ip::Sent::Pending(B))); + let out = toyos_net_ip::UdpOut { source: A, destination: B, ttl: toyos_net_wire::ipv4::Ttl::DEFAULT, broadcast: false, iface: None, datagram }; + assert_eq!(h.ip.send_udp(late, &usize::MAX, &out, &mut frame), Ok(toyos_net_ip::Sent::Pending { iface: h.if0, next_hop: B })); assert_eq!(h.ip.transmit(late, usize::MAX, |_, _| {}), 1); assert!(matches!(h.state(B), Some(Nud::Incomplete(_)))); assert_eq!(h.ip.next_deadline(), Some(Instant::from_nanos(u64::MAX))); diff --git a/system/netstack/ip/tests/common/mod.rs b/system/netstack/ip/tests/common/mod.rs index 01cb60a2879..09883196a35 100644 --- a/system/netstack/ip/tests/common/mod.rs +++ b/system/netstack/ip/tests/common/mod.rs @@ -7,7 +7,7 @@ use std::net::Ipv4Addr; -use toyos_net_ip::{AddrState, Counter, Event, IfIndex, Instant, Ip, Nud, Sent, UdpOut, FRAME}; +use toyos_net_ip::{AddrState, Counter, Event, IfIndex, Instant, Ip, Nud, Rank, Sent, UdpOut, FRAME}; use toyos_net_wire::ipv4::Ttl; use toyos_net_wire::udp::UdpBuilder; use toyos_net_wire::Port; @@ -217,7 +217,7 @@ impl H { /// Interface `if0` with MAC A and its link up at time 0, nothing settled. pub fn raw() -> Self { let mut ip = Ip::new(Instant::from_nanos(0), SECRET); - let if0 = ip.add_interface(Instant::from_nanos(0), mac(MAC_A)); + let if0 = ip.add_interface(Instant::from_nanos(0), mac(MAC_A), Rank(0)); ip.link_up(Instant::from_nanos(0), if0).unwrap(); Self { ip, if0, now: 0, events: Vec::new(), counters: Vec::new(), wire: std::collections::BTreeMap::default() } } @@ -300,7 +300,7 @@ impl H { let reply = eth(MAC_A, m, 0x0806, &arp_packet(2, m, addr, MAC_A, A)); let _ = h.ip.receive(setup, h.if0, &reply); h.settle(); - h.ip.advise(h.clock(), addr, toyos_net_ip::Advice::Confirmed); + h.ip.advise(h.clock(), addr, toyos_net_ip::Source::Any, toyos_net_ip::Advice::Confirmed); assert!(matches!(h.ip.neighbour(h.if0, addr), Some(Nud::Reachable(r)) if r.confirmed() == H::instant(0))); h.rebase(); h @@ -745,12 +745,13 @@ impl H { pub fn send_as(&mut self, source: Ipv4Addr, destination: Ipv4Addr, sport: u16, dport: u16, data: &[u8], ttl: Ttl, broadcast: bool) -> Result>, Counter> { let mut frame = [0u8; FRAME]; let datagram = UdpBuilder { source: Port::new(sport).unwrap(), destination: Port::new(dport).unwrap(), data }; - let out = UdpOut { source, destination, ttl, broadcast, datagram }; - let sent = self.ip.send_udp(self.clock(), &out, &mut frame); + let out = UdpOut { source, destination, ttl, broadcast, iface: None, datagram }; + let sent = self.ip.send_udp(self.clock(), &usize::MAX, &out, &mut frame); self.collect(); sent.map(|s| match s { - Sent::Frame(n) => Some(frame[..n].to_vec()), - Sent::Pending(_) => None, + Sent::Frame { len: n, .. } => Some(frame[..n].to_vec()), + Sent::Pending { .. } => None, + Sent::NoRoom { .. } => unreachable!("every interface has room"), }) } @@ -788,7 +789,7 @@ pub const MAC_A1: MacAddr = MacAddr([2, 0, 0, 0, 0, 0x1a]); impl H { /// A second interface with its own MAC and link up, at setup time 0. pub fn add_if1(&mut self) -> IfIndex { - let if1 = self.ip.add_interface(Instant::from_nanos(0), mac(MAC_A1)); + let if1 = self.ip.add_interface(Instant::from_nanos(0), mac(MAC_A1), Rank(0)); self.ip.link_up(Instant::from_nanos(0), if1).unwrap(); if1 } @@ -796,7 +797,7 @@ impl H { /// ToyOS as B: MAC B, 192.0.2.2/24, A resolved at setup. pub fn as_b() -> Self { let mut ip = Ip::new(Instant::from_nanos(0), SECRET); - let if0 = ip.add_interface(Instant::from_nanos(0), mac(MAC_B)); + let if0 = ip.add_interface(Instant::from_nanos(0), mac(MAC_B), Rank(0)); ip.link_up(Instant::from_nanos(0), if0).unwrap(); let mut h = Self { ip, if0, now: 0, events: Vec::new(), counters: Vec::new(), wire: std::collections::BTreeMap::default() }; h.assign(if0, B, 24, 0); @@ -826,7 +827,7 @@ impl H { /// A host with MAC `m` holding `addr/len`, assigned through conflict detection, nothing learned. pub fn host(m: MacAddr, addr: Ipv4Addr, len: u8) -> Self { let mut ip = Ip::new(Instant::from_nanos(0), SECRET); - let if0 = ip.add_interface(Instant::from_nanos(0), mac(m)); + let if0 = ip.add_interface(Instant::from_nanos(0), mac(m), Rank(0)); ip.link_up(Instant::from_nanos(0), if0).unwrap(); let mut h = Self { ip, if0, now: 0, events: Vec::new(), counters: Vec::new(), wire: std::collections::BTreeMap::default() }; h.assign(if0, addr, len, 0); diff --git a/system/netstack/ip/tests/nud.rs b/system/netstack/ip/tests/nud.rs index 60109b1bf81..71a2f0ff0d3 100644 --- a/system/netstack/ip/tests/nud.rs +++ b/system/netstack/ip/tests/nud.rs @@ -123,7 +123,7 @@ fn s_ip_nud_007_positive_advice_moves_the_confirmation() { let reachable = h.ip.reachable_time(h.if0).unwrap(); let ms = |d: std::time::Duration| d.as_millis() as u64; h.at(10_000); - h.ip.advise(h.clock(), REMOTE, Advice::Confirmed); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Confirmed); h.run(ms(reachable) + 1); assert!(h.is_reachable(R), "the original deadline re-arms"); h.run(10_000 + ms(reachable) - 1); @@ -152,7 +152,7 @@ fn s_ip_nud_009_advice_ends_delay() { h.stale(R, MAC_R); h.udp_to(REMOTE).unwrap(); h.at(4_000); - h.ip.advise(h.clock(), REMOTE, Advice::Confirmed); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Confirmed); assert!(h.is_reachable(R)); assert!(h.run(5_000).is_empty()); } @@ -161,7 +161,7 @@ fn s_ip_nud_009_advice_ends_delay() { fn s_ip_nud_010_a_reply_confirms_probe() { let mut h = H::fixture_i_with(R, MAC_R); h.at(100); - h.ip.advise(h.clock(), REMOTE, Advice::Reverify); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(R), Some(Nud::Probe(_)))); h.out(); h.frame(&hex(V_ARP_REPLY_R)); @@ -261,10 +261,10 @@ fn s_ip_nud_015_an_assertion_revives_failed() { fn s_ip_nud_016_requests_are_spaced() { let mut h = H::fixture_i_with(R, MAC_R); h.at(100); - h.ip.advise(h.clock(), REMOTE, Advice::Reverify); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Reverify); let first = h.out(); h.at(600); - h.ip.advise(h.clock(), REMOTE, Advice::Reverify); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Reverify); let second = h.out(); let later = h.run(1_100); assert_eq!(first.iter().map(|o| o.frame.clone()).collect::>(), [hex(V_ARP_POLL_R)]); @@ -279,7 +279,7 @@ fn s_ip_nud_016_requests_are_spaced() { h.reply_from(R, MAC_R); assert!(h.is_reachable(R)); h.at(100); - h.ip.advise(h.clock(), R, Advice::Reverify); + h.ip.advise(h.clock(), R, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(R), Some(Nud::Probe(_)))); assert!(h.out().is_empty(), "100 ms after the last request"); let later = h.run(1_000); @@ -290,7 +290,7 @@ fn s_ip_nud_016_requests_are_spaced() { fn s_ip_nud_017_negative_advice_probes() { let mut h = H::fixture_i_with(R, MAC_R); h.at(100); - h.ip.advise(h.clock(), REMOTE, Advice::Reverify); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(R), Some(Nud::Probe(_)))); assert_eq!(h.out().iter().map(|o| o.frame.clone()).collect::>(), [hex(V_ARP_POLL_R)]); } @@ -299,10 +299,10 @@ fn s_ip_nud_017_negative_advice_probes() { fn s_ip_nud_018_negative_advice_leaves_probe_incomplete_and_failed() { let mut h = H::fixture_i_with(R, MAC_R); h.at(100); - h.ip.advise(h.clock(), REMOTE, Advice::Reverify); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Reverify); h.out(); h.at(200); - h.ip.advise(h.clock(), REMOTE, Advice::Reverify); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(R), Some(Nud::Probe(p)) if p.requests() == 1)); assert!(h.out().is_empty()); @@ -310,12 +310,12 @@ fn s_ip_nud_018_negative_advice_leaves_probe_incomplete_and_failed() { h.ip.set_gateways(h.clock(), h.if0, &[]).unwrap(); h.udp_to(B).unwrap(); h.out(); - h.ip.advise(h.clock(), B, Advice::Reverify); + h.ip.advise(h.clock(), B, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(B), Some(Nud::Incomplete(s)) if s.requests() == 1)); assert!(h.out().is_empty()); h.run(3_000); h.at(3_500); - h.ip.advise(h.clock(), B, Advice::Reverify); + h.ip.advise(h.clock(), B, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(B), Some(Nud::Failed))); assert!(h.out().is_empty()); } @@ -325,14 +325,14 @@ fn s_ip_nud_019_positive_advice_needs_a_mac() { let mut h = H::fixture_i(); h.udp_to(B).unwrap(); h.out(); - h.ip.advise(h.clock(), B, Advice::Confirmed); + h.ip.advise(h.clock(), B, toyos_net_ip::Source::Any, Advice::Confirmed); assert!(matches!(h.state(B), Some(Nud::Incomplete(_)))); h.run(3_000); - h.ip.advise(h.clock(), B, Advice::Confirmed); + h.ip.advise(h.clock(), B, toyos_net_ip::Source::Any, Advice::Confirmed); assert!(matches!(h.state(B), Some(Nud::Failed))); let (mut h, t0) = unreachable_r(); h.at(t0 + 10); - h.ip.advise(h.clock(), REMOTE, Advice::Confirmed); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Confirmed); assert!(h.is_reachable(R)); assert_eq!(h.mac_of(R), Some(MAC_R)); } @@ -341,7 +341,7 @@ fn s_ip_nud_019_positive_advice_needs_a_mac() { fn s_ip_nud_020_advice_lands_on_the_gateway() { let mut h = H::fixture_i_with(R, MAC_R); h.at(100); - h.ip.advise(h.clock(), REMOTE, Advice::Reverify); + h.ip.advise(h.clock(), REMOTE, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(R), Some(Nud::Probe(_)))); assert!(h.state(REMOTE).is_none()); } @@ -404,7 +404,7 @@ fn s_ip_nud_022_a_full_table_evicts_in_order() { let (failed, unreachable, stale) = (neighbour(0), neighbour(1), neighbour(2)); let _ = h.ip.resolve(h.clock(), h.if0, failed, A); reach_wide(&mut h, unreachable); - h.ip.advise(h.clock(), unreachable, Advice::Reverify); + h.ip.advise(h.clock(), unreachable, toyos_net_ip::Source::Any, Advice::Reverify); ask_wide(&mut h, stale); h.run(5_000); assert!(matches!(h.state(failed), Some(Nud::Failed))); @@ -742,7 +742,7 @@ fn b_holds_two_behind_a_full_queue() -> H { /// requests, each lost to the full queue, leave B quiescent UNREACHABLE from 4,000. fn unreachable_b(h: &mut H) { h.frame(&hex(V_ARP_REPLY)); - h.ip.advise(h.clock(), B, Advice::Reverify); + h.ip.advise(h.clock(), B, toyos_net_ip::Source::Any, Advice::Reverify); for t in [1_000, 2_000, 3_000, 4_000] { h.ip.fire(H::instant(t)); } @@ -789,7 +789,7 @@ fn s_ip_nud_030_released_datagrams_keep_their_entry() { h.at(5); h.frame(&hex(V_ARP_REPLY)); h.at(1_000); - h.ip.advise(h.clock(), B, Advice::Reverify); + h.ip.advise(h.clock(), B, toyos_net_ip::Source::Any, Advice::Reverify); assert!(matches!(h.state(B), Some(Nud::Probe(_)))); h.frame(ð(MacAddr::BROADCAST, MAC_X, 0x0806, &arp_packet(1, MAC_X, B, MacAddr::ZERO, B))); h.ip.fire(H::instant(601_000)); diff --git a/system/netstack/ip/tests/rank.rs b/system/netstack/ip/tests/rank.rs new file mode 100644 index 00000000000..e4e017be836 --- /dev/null +++ b/system/netstack/ip/tests/rank.rs @@ -0,0 +1,215 @@ +//! Interfaces that tie are chosen by rank, then by when each was added: the +//! default route, failover to the next as one goes down and its return. + +mod common; + +use std::net::Ipv4Addr; + +use common::*; +use toyos_net_ip::{Counter, IfIndex, Instant, Ip, NextHop, Rank, Route, Source}; +use toyos_net_wire::ethernet::MacAddr; + +const WIRED: Rank = Rank(0); +const WIRELESS: Rank = Rank(1); + +/// One interface's address and its gateway, on a /24 of its own. +struct Net { + mac: MacAddr, + addr: Ipv4Addr, + gateway: Ipv4Addr, +} + +const NET_1: Net = Net { mac: MacAddr([2, 0, 0, 0, 1, 1]), addr: Ipv4Addr::new(192, 0, 2, 10), gateway: Ipv4Addr::new(192, 0, 2, 1) }; +const NET_2: Net = Net { mac: MacAddr([2, 0, 0, 0, 2, 2]), addr: Ipv4Addr::new(198, 51, 100, 10), gateway: Ipv4Addr::new(198, 51, 100, 1) }; +const NET_3: Net = Net { mac: MacAddr([2, 0, 0, 0, 3, 3]), addr: Ipv4Addr::new(203, 0, 113, 10), gateway: Ipv4Addr::new(203, 0, 113, 1) }; + +/// Somewhere only a gateway reaches. +const OFF_LINK: Ipv4Addr = Ipv4Addr::new(192, 88, 99, 1); + +/// Each net's interface added in turn with its rank, up, addressed and given +/// its gateway. +fn machine(nets: &[(&Net, Rank)]) -> (Ip, Vec, Instant) { + let mut ip = Ip::new(Instant::from_nanos(0), SECRET); + let mut now = Instant::from_millis(0); + let mut ifaces = Vec::new(); + for &(net, rank) in nets { + let iface = ip.add_interface(now, mac(net.mac), rank); + ip.link_up(now, iface).unwrap(); + ip.add_address(now, iface, net.addr, 24).unwrap(); + ifaces.push(iface); + } + let settled = Instant::from_millis(10_000); + while let Some(deadline) = ip.next_deadline().filter(|d| *d <= settled) { + ip.fire(deadline); + ip.transmit(deadline, usize::MAX, |_, _| {}); + now = deadline; + } + for (&(net, _), &iface) in nets.iter().zip(&ifaces) { + ip.set_gateways(now, iface, &[net.gateway]).unwrap(); + } + (ip, ifaces, settled) +} + +fn via(ip: &mut Ip, destination: Ipv4Addr) -> Result { + ip.route(destination, Source::Any, None) +} + +fn by(iface: IfIndex, net: &Net) -> Result { + Ok(Route { iface, next_hop: NextHop::Neighbour(net.gateway), source: net.addr }) +} + +#[test] +fn the_default_route_leaves_by_the_lower_rank_whichever_came_first() { + let (mut ip, ifaces, _) = machine(&[(&NET_1, WIRELESS), (&NET_2, WIRED)]); + assert_eq!(via(&mut ip, OFF_LINK), by(ifaces[1], &NET_2)); +} + +#[test] +fn among_equal_ranks_the_first_added_leads() { + let (mut ip, ifaces, _) = machine(&[(&NET_1, WIRELESS), (&NET_2, WIRED), (&NET_3, WIRED)]); + assert_eq!(via(&mut ip, OFF_LINK), by(ifaces[1], &NET_2)); + let (mut ip, ifaces, _) = machine(&[(&NET_3, WIRED), (&NET_2, WIRED)]); + assert_eq!(via(&mut ip, OFF_LINK), by(ifaces[0], &NET_3)); +} + +#[test] +fn a_link_that_goes_down_hands_the_default_route_on_and_takes_it_back_on_return() { + let (mut ip, ifaces, now) = machine(&[(&NET_1, WIRELESS), (&NET_2, WIRED)]); + ip.link_down(now, ifaces[1]).unwrap(); + assert_eq!(via(&mut ip, OFF_LINK), by(ifaces[0], &NET_1), "failover to the wireless link"); + ip.link_up(now, ifaces[1]).unwrap(); + assert_eq!(via(&mut ip, OFF_LINK), by(ifaces[1], &NET_2), "the wired link's return"); + ip.link_down(now, ifaces[0]).unwrap(); + ip.link_down(now, ifaces[1]).unwrap(); + assert_eq!(via(&mut ip, OFF_LINK), Err(Counter::RouteNoSourceAddress), "no link up"); +} + +#[test] +fn a_connected_prefix_beats_a_lower_ranks_gateway() { + let (mut ip, ifaces, _) = machine(&[(&NET_1, WIRELESS), (&NET_2, WIRED)]); + let neighbour = Ipv4Addr::new(192, 0, 2, 77); + assert_eq!(via(&mut ip, neighbour), Ok(Route { iface: ifaces[0], next_hop: NextHop::Neighbour(neighbour), source: NET_1.addr })); +} + +#[test] +fn a_bound_source_keeps_its_own_interface_whatever_its_rank() { + let (mut ip, ifaces, now) = machine(&[(&NET_1, WIRELESS), (&NET_2, WIRED)]); + assert_eq!(ip.route(OFF_LINK, Source::Bound(NET_1.addr), None), by(ifaces[0], &NET_1)); + // Its link down, the flow has no route, and does not move to the other address. + ip.link_down(now, ifaces[0]).unwrap(); + assert_eq!(ip.route(OFF_LINK, Source::Bound(NET_1.addr), None), Err(Counter::RouteNoSourceAddress)); +} + +/// Room on one interface alone. +struct Only(IfIndex, usize); + +impl toyos_net_ip::Credit for Only { + fn room(&self, iface: IfIndex) -> bool { + iface == self.0 && self.1 > 0 + } + fn spend(&mut self, _: IfIndex) { + self.1 -= 1; + } +} + +#[test] +fn a_link_without_room_holds_back_no_other_links_frames() { + let (mut ip, ifaces, now) = machine(&[(&NET_1, WIRED), (&NET_2, WIRED)]); + // An ARP request queued on each, the first's ahead. + ip.resolve(now, ifaces[0], NET_1.gateway, NET_1.addr); + ip.resolve(now, ifaces[1], NET_2.gateway, NET_2.addr); + let mut out = Vec::new(); + assert_eq!(ip.transmit(now, Only(ifaces[1], 8), |iface, _| out.push(iface)), 1); + assert_eq!(out, [ifaces[1]], "the second link's request, past the first's"); + out.clear(); + assert_eq!(ip.transmit(now, 8usize, |iface, _| out.push(iface)), 1); + assert_eq!(out, [ifaces[0]], "the first's, still queued"); +} + +// ---- one LAN on two links ---- + +/// The wired and wireless links bridged onto one LAN: one prefix, one gateway address. +const LAN_GATEWAY: Ipv4Addr = Ipv4Addr::new(192, 168, 1, 1); +const LAN_WIRED: Net = Net { mac: MacAddr([2, 0, 0, 0, 4, 4]), addr: Ipv4Addr::new(192, 168, 1, 10), gateway: LAN_GATEWAY }; +const LAN_WIRELESS: Net = Net { mac: MacAddr([2, 0, 0, 0, 5, 5]), addr: Ipv4Addr::new(192, 168, 1, 20), gateway: LAN_GATEWAY }; +const GATEWAY_MAC: MacAddr = MacAddr([2, 0, 0, 0, 0, 0xfe]); + +/// The gateway's ARP reply to `net`. +fn gateway_reply(net: &Net) -> Vec { + use toyos_net_wire::arp::{Arp, Operation}; + use toyos_net_wire::ethernet::FrameBuilder; + let arp = Arp { operation: Operation::Reply, sender_mac: GATEWAY_MAC, sender_ip: LAN_GATEWAY, target_mac: net.mac, target_ip: net.addr }; + let mut frame = vec![0; 64]; + let len = FrameBuilder { destination: net.mac, source: mac(GATEWAY_MAC) }.emit(&arp, &mut frame).unwrap().len(); + frame.truncate(len); + frame +} + +/// A stale entry: resolved, then left past any reachable time. +fn stale(ip: &mut Ip, iface: IfIndex, net: &Net, now: Instant) -> Instant { + ip.resolve(now, iface, LAN_GATEWAY, net.addr); + ip.transmit(now, usize::MAX, |_, _| {}); + ip.receive(now, iface, &gateway_reply(net)); + let later = now.after(std::time::Duration::from_secs(120)); + while let Some(deadline) = ip.next_deadline().filter(|d| *d <= later) { + ip.fire(deadline); + ip.transmit(deadline, usize::MAX, |_, _| {}); + } + later +} + +#[test] +fn a_flows_advice_lands_on_its_own_links_gateway_where_two_links_share_one() { + let (mut ip, ifaces, now) = machine(&[(&LAN_WIRED, WIRED), (&LAN_WIRELESS, WIRELESS)]); + let now = stale(&mut ip, ifaces[0], &LAN_WIRED, now); + let now = stale(&mut ip, ifaces[1], &LAN_WIRELESS, now); + let state = |ip: &Ip, iface| ip.neighbour(iface, LAN_GATEWAY).map(|n| matches!(n, toyos_net_ip::Nud::Reachable(_))); + assert_eq!((state(&ip, ifaces[0]), state(&ip, ifaces[1])), (Some(false), Some(false)), "the premise: both stale"); + // A flow on the wireless link, from its address, confirms its next hop. + ip.advise(now, OFF_LINK, Source::Bound(LAN_WIRELESS.addr), toyos_net_ip::Advice::Confirmed); + assert_eq!((state(&ip, ifaces[0]), state(&ip, ifaces[1])), (Some(false), Some(true)), "the wireless entry, not the wired one"); +} + +/// An ARP request from host `n` on `net`'s /24 for `net`'s address. +fn asks_for(net: &Net, n: u8) -> Vec { + use toyos_net_wire::arp::{Arp, Operation}; + use toyos_net_wire::ethernet::FrameBuilder; + let [a, b, c, _] = net.addr.octets(); + let host = MacAddr([2, 9, 0, 0, 0, n]); + let arp = Arp { operation: Operation::Request, sender_mac: host, sender_ip: Ipv4Addr::new(a, b, c, 100u8.wrapping_add(n)), target_mac: MacAddr::ZERO, target_ip: net.addr }; + let mut frame = vec![0; 64]; + let len = FrameBuilder { destination: MacAddr::BROADCAST, source: mac(host) }.emit(&arp, &mut frame).unwrap().len(); + frame.truncate(len); + frame +} + +#[test] +fn a_link_that_takes_nothing_holds_back_none_of_another_links_own_frames() { + let (mut ip, ifaces, now) = machine(&[(&NET_1, WIRED), (&NET_2, WIRELESS)]); + let (stuck, other) = (ifaces[0], ifaces[1]); + // The wired link's share of the queue overfilled with its replies, none of which leave. + let full_before = ip.counters().get(Counter::IpControlQueueFull); + for n in 0..(toyos_net_ip::limits::CONTROL_QUEUE as u8 + 8) { + ip.receive(now, stuck, &asks_for(&NET_1, n)); + } + assert_eq!(ip.counters().get(Counter::IpControlQueueFull), full_before + 8, "the stuck link's own overflow is its own"); + // The other link's request is queued and leaves, past the stuck link's share. + ip.resolve(now, other, NET_2.gateway, NET_2.addr); + let mut out = Vec::new(); + ip.transmit(now, Only(other, usize::MAX), |iface, _| out.push(iface)); + assert_eq!(out, [other], "its request"); + // A second address on the other link: its probes leave and conflict detection ends. + let second = Ipv4Addr::new(198, 51, 100, 11); + ip.add_address(now, other, second, 24).unwrap(); + let until = now.after(std::time::Duration::from_secs(10)); + let mut probes = 0; + while let Some(deadline) = ip.next_deadline().filter(|d| *d <= until) { + ip.fire(deadline); + ip.transmit(deadline, Only(other, usize::MAX), |iface, _| { + assert_eq!(iface, other); + probes += 1; + }); + } + assert!(probes >= 3, "the probes and announcements left: {probes}"); + assert_eq!(ip.address(other, second), Some(toyos_net_ip::AddrState::Assigned)); +} diff --git a/system/netstack/node/src/lease.rs b/system/netstack/node/src/lease.rs index f063d6a9315..64d3420028b 100644 --- a/system/netstack/node/src/lease.rs +++ b/system/netstack/node/src/lease.rs @@ -1,62 +1,34 @@ -//! The interface's address, gateway and resolvers as one state, and its one writer. +//! Each link's address, gateway and resolvers as one state, and its one writer. //! -//! The state is `Unaddressed`, `Probing` an address a server acknowledged, or `Held`: a lease -//! whose address [ip] verified. The shard is private to this module, so nothing else in the crate -//! reaches `add_address`, `remove_address` or `set_gateways`: +//! A link's state is `Unaddressed`, `Probing` an address a server acknowledged, or `Held`: a +//! lease whose address [ip] verified. The shard is private to this module, so nothing else in the +//! crate reaches `add_address`, `remove_address` or `set_gateways`: //! -//! - the gateway is written only with a held lease, and [ip] withdraws it with the address, so no -//! route outlives its address; +//! - a link's gateway is written only with its held lease, and [ip] withdraws it with the +//! address, so no route outlives its address; //! - a lease is held only against a [`Verified`], which only [`Stack::report`] makes, from [ip]'s -//! report for the address under probe, so no address is in use before conflict detection ends; -//! - the resolvers are the held lease's, so they come and go with it, and the answers they gave -//! go with every write of the state and every link that goes down ([`network::Network`]): after -//! either the next lookup may be asking another network, so none is answered from this one's. +//! report for the link's address under probe, so no address is in use before conflict +//! detection ends; +//! - the resolvers lookups ask are one link's held lease's: of the links up and holding one, the +//! first by rank and then arrival, and with none up the link asked before ([`Stack::asked`]). +//! The answers they gave go with every write of any link's state, every link that goes down +//! and every change of the link asked: after any of them the next lookup may be asking another +//! network, so none is answered from this one's. //! -//! [ip] reports on no address but the one this module added, and one report at a time: each is -//! read against the state as it stands. +//! [ip] reports on no address but those this module added, and one report at a time: each is +//! read against its link's state as it stands. mod tcp; -mod network { - use toyos_dns::Cache; - - use super::State; - - /// The state and the answers its resolvers gave, written together: the state only by - /// [`Network::set`], which empties the answers as it writes it. - pub(super) struct Network { - state: State, - answers: Cache, - } - - impl Network { - pub(super) fn new() -> Self { - Self { state: State::Unaddressed, answers: Cache::default() } - } - - pub(super) fn state(&self) -> &State { - &self.state - } - - pub(super) fn set(&mut self, state: State) { - self.state = state; - self.answers.empty(); - } - - pub(super) fn answers(&mut self) -> &mut Cache { - &mut self.answers - } - } -} - use alloc::collections::VecDeque; +use alloc::vec::Vec; use core::net::Ipv4Addr; use toyos_dhcp::{Lease, Transmission}; use toyos_dns::Cache; -use toyos_net_shard::{Config, Event, Refusal, Shard}; +use toyos_net_shard::{Config, Credit, Event, IfIndex, Origin, Rank, Refusal, Shard}; use toyos_net_udp::{Received, SocketId}; -use toyos_net_wire::ethernet::MacAddr; +use toyos_net_wire::ethernet::{IndividualMac, MacAddr}; use toyos_net_wire::ipv4::{MulticastAddr, Ttl}; use toyos_net_wire::{Instant, Port}; @@ -71,37 +43,52 @@ enum State { Held(Lease), } -/// Proof that [ip] finished conflict detection for the address under probe. +/// Proof that [ip] finished conflict detection for a link's address under probe. pub(crate) struct Verified(Ipv4Addr); -/// What [ip] said became of an address, one event at a time. +/// What [ip] said became of a link's address, one event at a time. pub(crate) enum Report { Verified(Verified), - /// Another host holds the address, at this MAC: [ip] removed it, and the state is + /// Another host holds the address, at this MAC: [ip] removed it, and the link's state is /// `Unaddressed`. Conflict(MacAddr), - /// The link went down under the probe: [ip] removed the address, and the state is + /// The link went down under the probe: [ip] removed the address, and the link's state is /// `Unaddressed`. NotVerified, } +/// One link's part. +struct Link { + iface: IfIndex, + rank: Rank, + up: bool, + state: State, +} + pub(crate) struct Stack { shard: Shard, - /// The DHCP client's socket. + /// The DHCP clients' one socket, each datagram for the client of the link it came by. socket: SocketId, - network: network::Network, + /// In the order they were added. + links: Vec, + /// The link lookups ask ([`Stack::asked`]), kept as each write of a link's state or link + /// leaves it. + asked: Option, + /// What the asked link's resolvers answered: emptied with every write of a link's state, + /// every link that goes down and every change of the link asked. + answers: Cache, /// What [ip] reported of an address and [`Stack::report`] has not read: never a log line. - inbox: VecDeque, + inbox: VecDeque<(IfIndex, Fate)>, } -/// [ip]'s report on the address this module added. +/// [ip]'s report on an address this module added. enum Fate { Verified(Ipv4Addr), /// Removed by [ip]: for the host at this MAC, or with none, for a link that went down. Taken(Ipv4Addr, Option), } -/// [ip] refuses nothing about the one interface and the one address this module gave it. +/// [ip] refuses nothing about an interface and an address this module gave it. fn agreed(answer: Result) -> T { match answer { Ok(value) => value, @@ -110,39 +97,95 @@ fn agreed(answer: Result) -> T { } impl Stack { - /// One interface, link down, unaddressed. + /// No link yet. pub(crate) fn new(now: Instant, config: Config) -> Result { let mut shard = Shard::new(now, config)?; let Ok(socket) = shard.acquisition() else { unreachable!("a new shard has no socket on port 68") }; - Ok(Self { shard, socket, network: network::Network::new(), inbox: VecDeque::new() }) + Ok(Self { shard, socket, links: Vec::new(), asked: None, answers: Cache::default(), inbox: VecDeque::new() }) + } + + /// A link's interface, down and unaddressed. + pub(crate) fn add_link(&mut self, now: Instant, mac: IndividualMac, rank: Rank) -> IfIndex { + let iface = self.shard.add_interface(now, mac, rank); + self.links.push(Link { iface, rank, up: false, state: State::Unaddressed }); + iface } pub(crate) fn shard(&self) -> &Shard { &self.shard } - pub(crate) fn lease(&self) -> Option<&Lease> { - match self.network.state() { + /// Every link, in the order added. + pub(crate) fn links(&self) -> impl Iterator + '_ { + self.links.iter().map(|link| link.iface) + } + + fn link_of(&self, iface: IfIndex) -> &Link { + self.links.iter().find(|link| link.iface == iface).unwrap_or_else(|| unreachable!("{iface:?} is no link of the node's")) + } + + fn link_mut(&mut self, iface: IfIndex) -> &mut Link { + self.links.iter_mut().find(|link| link.iface == iface).unwrap_or_else(|| unreachable!("{iface:?} is no link of the node's")) + } + + pub(crate) fn up(&self, iface: IfIndex) -> bool { + self.link_of(iface).up + } + + /// `iface`'s held lease. + pub(crate) fn lease(&self, iface: IfIndex) -> Option<&Lease> { + match &self.link_of(iface).state { State::Held(lease) => Some(lease), State::Unaddressed | State::Probing(_) => None, } } - fn address(&self) -> Option { - match self.network.state() { + /// The link lookups ask: of the links up and holding a lease, the first by rank and then by + /// when it was added. With none such, the link asked before while it still holds its lease, + /// since a link down keeps it; and failing that the first by rank holding one. + pub(crate) fn asked(&self) -> Option { + self.asked + } + + /// [`Self::asked`] chosen again after a link's state or link changed; `true` when it moved. + fn choose_asked(&mut self) -> bool { + let held = |link: &&Link| matches!(link.state, State::Held(_)); + let up = self.links.iter().filter(held).filter(|link| link.up).min_by_key(|link| link.rank); + let kept = self.asked.and_then(|asked| self.links.iter().filter(held).find(|link| link.iface == asked)); + let any = self.links.iter().filter(held).min_by_key(|link| link.rank); + let asked = up.or(kept).or(any).map(|link| link.iface); + let moved = asked != self.asked; + self.asked = asked; + moved + } + + /// The asked link's lease, whose resolvers lookups ask. + pub(crate) fn asked_lease(&self) -> Option<&Lease> { + self.asked().and_then(|iface| self.lease(iface)) + } + + fn address(&self, iface: IfIndex) -> Option { + match &self.link_of(iface).state { State::Unaddressed => None, State::Probing(address) => Some(*address), State::Held(lease) => Some(lease.address), } } + /// Writes `iface`'s state; the answers go with it. + fn set(&mut self, iface: IfIndex, state: State) { + self.link_mut(iface).state = state; + self.choose_asked(); + self.answers.empty(); + } + // ---- the device and the clock ---- - pub(crate) fn receive(&mut self, now: Instant, frame: &[u8]) { - self.shard.receive(now, frame); + pub(crate) fn receive(&mut self, now: Instant, iface: IfIndex, frame: &[u8]) { + self.shard.receive(now, iface, frame); } - pub(crate) fn transmit(&mut self, now: Instant, credit: usize, sink: impl FnMut(&[u8])) -> usize { + pub(crate) fn transmit(&mut self, now: Instant, credit: impl Credit, sink: impl FnMut(IfIndex, &[u8])) -> usize { self.shard.transmit(now, credit, sink) } @@ -155,34 +198,37 @@ impl Stack { } /// Down, the answers go and a held lease stays: the link may come back on another network - /// that the lease's verification never hears from. - pub(crate) fn link(&mut self, now: Instant, up: bool) { - agreed(if up { self.shard.link_up(now) } else { self.shard.link_down(now) }); - if !up { - self.network.answers().empty(); + /// that the lease's verification never hears from. Up or down, a change of the link asked + /// empties them too. + pub(crate) fn link(&mut self, now: Instant, iface: IfIndex, up: bool) { + agreed(if up { self.shard.link_up(now, iface) } else { self.shard.link_down(now, iface) }); + self.link_mut(iface).up = up; + if self.choose_asked() || !up { + self.answers.empty(); } } - /// What the held lease's resolvers answered, kept by the view that asked. + /// What the asked link's resolvers answered, kept by the view that asked. pub(crate) fn answers(&mut self) -> &mut Cache { - self.network.answers() + &mut self.answers } - // ---- the DHCP client's socket ---- + // ---- the DHCP clients' socket ---- - /// Queues a message of the client's; a refusal is [udp]'s, by the rule it counted. - pub(crate) fn send(&mut self, now: Instant, message: &Transmission) -> Result<(), toyos_net_udp::Error> { - self.shard.send_from(now, self.socket, message.source, message.destination, SERVER_PORT, &message.payload) + /// Queues a message of `iface`'s client; a refusal is [udp]'s, by the rule it counted. + pub(crate) fn send(&mut self, now: Instant, iface: IfIndex, message: &Transmission) -> Result<(), toyos_net_udp::Error> { + let origin = Origin { iface, source: message.source }; + self.shard.send_from(now, self.socket, origin, message.destination, SERVER_PORT, &message.payload) } - /// The oldest datagram that reached the client's socket, in `out`, and the address it came - /// from. - pub(crate) fn recv<'a>(&mut self, out: &'a mut [u8]) -> Option<(&'a [u8], Ipv4Addr)> { + /// The oldest datagram that reached the clients' socket, in `out`, the address it came from + /// and the link it came by. + pub(crate) fn recv<'a>(&mut self, out: &'a mut [u8]) -> Option<(&'a [u8], Ipv4Addr, IfIndex)> { let Ok(received) = self.shard.recv_from(self.socket, out) else { - unreachable!("the client's socket is never closed and never connected") + unreachable!("the clients' socket is never closed and never connected") }; let received = received?; - Some((out.get(..received.len)?, received.source)) + Some((out.get(..received.len)?, received.source, received.iface)) } // ---- datagram sockets: [udp]'s calls passed on, none of which writes the lease ---- @@ -243,15 +289,20 @@ impl Stack { self.shard.udp_set_ttl(id, unicast, multicast) } - pub(crate) fn join(&mut self, now: Instant, group: MulticastAddr) { - agreed(self.shard.join(now, group)); + /// Queues a datagram that leaves by `iface` alone. + pub(crate) fn send_via(&mut self, now: Instant, id: SocketId, iface: IfIndex, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), toyos_net_udp::Error> { + self.shard.send_via(now, id, iface, destination, port, payload) + } + + pub(crate) fn join(&mut self, now: Instant, iface: IfIndex, group: MulticastAddr) { + agreed(self.shard.join(now, iface, group)); } // ---- the lease ---- /// Takes what the shard has reported: each log line goes to `line`, ahead of the reports - /// before it since none is read against the state, and each report on the address waits its - /// turn for [`Self::report`]. + /// before it since none is read against a state, and each report on an address waits its turn + /// for [`Self::report`]. pub(crate) fn refusals(&mut self, mut line: impl FnMut(Refusal, u64)) { for event in self.shard.drain_events() { self.inbox.push_back(match event { @@ -259,84 +310,86 @@ impl Stack { line(refusal, suppressed); continue; } - Event::Verified(addr) => Fate::Verified(addr), - Event::Conflict { addr, mac } | Event::Lost { addr, mac } => Fate::Taken(addr, Some(mac)), - Event::NotVerified(addr) => Fate::Taken(addr, None), + Event::Verified { iface, addr } => (iface, Fate::Verified(addr)), + Event::Conflict { iface, addr, mac } | Event::Lost { iface, addr, mac } => (iface, Fate::Taken(addr, Some(mac))), + Event::NotVerified { iface, addr } => (iface, Fate::Taken(addr, None)), }); } } - /// The next report [`Self::refusals`] took, read against the state. - pub(crate) fn report(&mut self) -> Option { - Some(match self.inbox.pop_front()? { - Fate::Verified(addr) => match self.network.state() { - State::Probing(address) if *address == addr => Report::Verified(Verified(addr)), + /// The next report [`Self::refusals`] took, read against its link's state, and the link. + pub(crate) fn report(&mut self) -> Option<(IfIndex, Report)> { + let (iface, fate) = self.inbox.pop_front()?; + Some((iface, match fate { + Fate::Verified(addr) => match self.link_of(iface).state { + State::Probing(address) if address == addr => Report::Verified(Verified(addr)), _ => unreachable!("[ip] verified {addr}, which is not under probe"), }, Fate::Taken(addr, by) => { - self.taken(addr); + self.taken(iface, addr); by.map_or(Report::NotVerified, Report::Conflict) } - }) + })) } /// [ip] removed `addr` itself, its gateways with it. - fn taken(&mut self, addr: Ipv4Addr) { - if self.address() != Some(addr) { - unreachable!("[ip] removed {addr}, which the lease does not name"); + fn taken(&mut self, iface: IfIndex, addr: Ipv4Addr) { + if self.address(iface) != Some(addr) { + unreachable!("[ip] removed {addr}, which the link's lease does not name"); } - self.network.set(State::Unaddressed); + self.set(iface, State::Unaddressed); } - /// Nothing is held: the address goes, and [ip] withdraws the gateway with it. - pub(crate) fn release(&mut self, now: Instant) { - if let Some(address) = self.address() { - agreed(self.shard.remove_address(now, address)); + /// Nothing is held on `iface`: its address goes, and [ip] withdraws the gateway and ends the + /// TCP connections with it. + pub(crate) fn release(&mut self, now: Instant, iface: IfIndex) { + if let Some(address) = self.address(iface) { + agreed(self.shard.remove_address(now, iface, address)); } - self.network.set(State::Unaddressed); + self.set(iface, State::Unaddressed); } - /// A server acknowledged `address`: [ip] probes it, and nothing uses it yet. Refused, the - /// state is `Unaddressed`. - pub(crate) fn probe(&mut self, now: Instant, address: Ipv4Addr, prefix_len: u8) -> Result<(), toyos_net_ip::Counter> { - self.release(now); - self.shard.add_address(now, address, prefix_len)?; - self.network.set(State::Probing(address)); + /// A server acknowledged `address` on `iface`: [ip] probes it, and nothing uses it yet. + /// Refused, the link's state is `Unaddressed`. + pub(crate) fn probe(&mut self, now: Instant, iface: IfIndex, address: Ipv4Addr, prefix_len: u8) -> Result<(), toyos_net_ip::Counter> { + self.release(now, iface); + self.shard.add_address(now, iface, address, prefix_len)?; + self.set(iface, State::Probing(address)); Ok(()) } /// The verified address's lease is in use from here: [ip] holds the address at the prefix /// `probe` gave it, and the router is written with the lease. Returns whether [ip] refused the /// router. - pub(crate) fn hold(&mut self, now: Instant, verified: Verified, mut lease: Lease) -> bool { + pub(crate) fn hold(&mut self, now: Instant, iface: IfIndex, verified: Verified, mut lease: Lease) -> bool { if lease.address != verified.0 { unreachable!("the client configured {}, having probed {}", lease.address, verified.0); } - let refused = self.route(now, &mut lease); - self.network.set(State::Held(lease)); + let refused = self.route(now, iface, &mut lease); + self.set(iface, State::Held(lease)); refused } /// The held address acknowledged again, with new times or a new prefix, router or resolvers. /// Returns whether [ip] refused its router. A prefix [ip] refuses changes nothing: the lease /// stays as it was held. - pub(crate) fn renew(&mut self, now: Instant, mut lease: Lease) -> Result { - if !matches!(self.network.state(), State::Held(held) if held.address == lease.address) { + pub(crate) fn renew(&mut self, now: Instant, iface: IfIndex, mut lease: Lease) -> Result { + if !matches!(&self.link_of(iface).state, State::Held(held) if held.address == lease.address) { unreachable!("the client renewed {}, which is not held", lease.address); } - self.shard.add_address(now, lease.address, lease.prefix_len)?; - let refused = self.route(now, &mut lease); - self.network.set(State::Held(lease)); + self.shard.add_address(now, iface, lease.address, lease.prefix_len)?; + let refused = self.route(now, iface, &mut lease); + self.set(iface, State::Held(lease)); Ok(refused) } /// Writes `lease`'s router, which [ip] takes only on a usable prefix. One [ip] refuses is no /// router: the lease is left without one, and so is [ip]. Returns whether it was refused. - fn route(&mut self, now: Instant, lease: &mut Lease) -> bool { - let refused = self.shard.set_gateways(now, lease.router.as_slice()).is_err(); + fn route(&mut self, now: Instant, iface: IfIndex, lease: &mut Lease) -> bool { + let refused = self.shard.set_gateways(now, iface, lease.router.as_slice()).is_err(); if refused { // [ip] kept the list it had. - agreed(self.shard.set_gateways(now, &[])); + agreed(self.shard.set_gateways(now, iface, &[])); lease.router = None; } refused @@ -368,11 +421,12 @@ mod tests { }; let Some(mac) = IndividualMac::new(MacAddr([2, 0, 0, 0, 0, 0x0a])) else { unreachable!("an individual address") }; let mut now = Instant::from_millis(3_600_000); - let Ok(mut stack) = Stack::new(now, Config { mac, receive_buffer: 65_535, send_buffer: 65_535, secrets }) else { unreachable!("a configuration [tcp] takes") }; - stack.link(now, true); - assert_eq!(stack.probe(now, A, 24), Ok(())); - while stack.shard.ip().address(stack.shard.iface(), A) == Some(toyos_net_ip::AddrState::Tentative) { - stack.transmit(now, usize::MAX, |_| {}); + let Ok(mut stack) = Stack::new(now, Config { receive_buffer: 65_535, send_buffer: 65_535, secrets }) else { unreachable!("a configuration [tcp] takes") }; + let link = stack.add_link(now, mac, Rank(0)); + stack.link(now, link, true); + assert_eq!(stack.probe(now, link, A, 24), Ok(())); + while stack.shard.ip().address(link, A) == Some(toyos_net_ip::AddrState::Tentative) { + stack.transmit(now, usize::MAX, |_, _| {}); let Some(next) = stack.next_deadline() else { unreachable!("conflict detection is still running") }; now = next; stack.fire(now); @@ -383,7 +437,7 @@ mod tests { let mut lines = Vec::new(); stack.refusals(|refusal, _| lines.push(refusal)); assert!(matches!(lines.as_slice(), [Refusal::Udp(refusal)] if refusal.rule == Counter::SendUnspecifiedDestination), "{lines:?}"); - assert!(matches!(stack.report(), Some(Report::Verified(Verified(A)))), "the verification is still to be read"); + assert!(matches!(stack.report(), Some((_, Report::Verified(Verified(A))))), "the verification is still to be read"); assert!(stack.report().is_none()); stack.refusals(|refusal, _| lines.push(refusal)); assert_eq!((lines.len(), stack.report().is_none()), (1, true), "each was handed over once"); diff --git a/system/netstack/node/src/lib.rs b/system/netstack/node/src/lib.rs index 417d1e39e0b..7ca9ccc85fa 100644 --- a/system/netstack/node/src/lib.rs +++ b/system/netstack/node/src/lib.rs @@ -1,10 +1,10 @@ -//! Netstack's decisions, pure: one interface of ToyOS's own stack (`toyos-net-shard`) and the -//! DHCP client (`toyos-dhcp`) that addresses it. Frames and time come in; frames leave only inside -//! [`Node::transmit`], against the credit the device offers, so there is no frame without a -//! transmit slot. Nothing here reads a clock, draws randomness or does I/O: the caller hands in +//! Netstack's decisions, pure: ToyOS's own stack (`toyos-net-shard`) over every link a driver +//! brings, each with the DHCP client (`toyos-dhcp`) that addresses it. Frames and time come in; +//! frames leave only inside [`Node::transmit`], against the credit each link offers, so there is +//! no frame without a transmit slot. Nothing here reads a clock, draws randomness or does I/O: the caller hands in //! the time, every draw and the shard's secrets. //! -//! The node carries out what the client asks and tells it what became of its address. What the +//! The node carries out what each link's client asks and tells it what became of its address. What the //! interface holds is `lease`'s to write and nobody else's: see that module for the three rules //! its types keep. A client's TCP connection and the region it lent are `streams`', a passive open //! and its owner's wakes `listeners`'. @@ -55,7 +55,9 @@ use alloc::vec; use alloc::vec::Vec; use toyos_dhcp::{AddressRequest, Client, HostName, Lease, Output}; +pub use toyos_net_shard::{Credit, IfIndex, Rank}; use toyos_net_shard::{Config, Shard}; +use toyos_net_wire::ethernet::IndividualMac; use toyos_net_wire::Instant; pub use datagram::{Datagram, DatagramId, Refused}; @@ -64,6 +66,7 @@ pub use resolve::{Ended, LookupId, NotStarted, Resolved, Resolving}; toyos_net_wire::counters! { DhcpUnsent = "node.dhcp-unsent"; + DhcpOtherLink = "node.dhcp-other-link"; AddressRefused = "node.address-refused"; RouterRefused = "node.router-refused"; NameUnsent = "node.name-unsent"; @@ -76,25 +79,34 @@ toyos_net_wire::counters! { pub enum Event { /// A refusal of the stack's, and how many of its rule it stands for beyond itself. Stack { refusal: toyos_net_shard::Refusal, suppressed: u64 }, - /// A refusal of the client's, and how many of its rule it stands for beyond itself. - Dhcp { refusal: toyos_dhcp::Refusal, suppressed: u64 }, - /// What became of the machine's name. - Name(toyos_mdns::Event), + /// A refusal of a link's client's, and how many of its rule it stands for beyond itself. + Dhcp { link: IfIndex, refusal: toyos_dhcp::Refusal, suppressed: u64 }, + /// What became of the machine's name on a link. + Name { link: IfIndex, event: toyos_mdns::Event }, +} + +/// One link's DHCP client. +struct LinkClient { + link: IfIndex, + mac: IndividualMac, + client: Client, + /// Its refusals pass through it as the stack's pass through the shard's: one line a rule in + /// any 10 s, carrying how many were not. + log: toyos_dhcp::RefusalLog, } pub struct Node { stack: Stack, - client: Client, - /// The responder for the machine's name, once [`Self::answer_as`] started it. + /// The name each link's client asks its network to record. + host_name: Option, + /// Each link's DHCP client, its link's address and the log its refusals pass through, in + /// the order the links were added. + clients: Vec, + /// The responders for the machine's name, once [`Self::answer_as`] started them. name: Option, - /// The link as [`Self::link`] last reported it. - up: bool, resolver: resolve::Resolver, counters: Counters, events: Vec, - /// The client's refusals pass through it as the stack's pass through the shard's: one line a - /// rule in any 10 s, carrying how many were not. - dhcp_log: toyos_dhcp::RefusalLog, /// Room for the largest message the client accepts. datagram: Vec, streams: streams::Streams, @@ -107,22 +119,17 @@ pub struct Node { } impl Node { - /// One interface, link down and unaddressed, with no place for a stream or a listener until - /// [`Self::set_places`]. The client begins here, and its first DISCOVER - /// is not sent: an exchange starts over when the link comes up ([`Self::link`]). - pub fn new(now: Instant, config: Config, host_name: Option, draw: impl FnMut() -> u32) -> Result { - let mac = config.mac; - let stack = Stack::new(now, config)?; - let (client, _unsent) = Client::start(now, mac, host_name, draw); + /// No link, and no place for a stream or a listener until [`Self::set_places`]. It serves + /// from here: what needs a link is refused for want of a route until one comes. + pub fn new(now: Instant, config: Config, host_name: Option) -> Result { Ok(Self { - stack, - client, + stack: Stack::new(now, config)?, + host_name, + clients: Vec::new(), name: None, - up: false, resolver: resolve::Resolver::new(), counters: Counters::default(), events: Vec::new(), - dhcp_log: toyos_dhcp::RefusalLog::default(), datagram: vec![0; usize::from(toyos_dhcp::limits::MAX_MESSAGE)], streams: streams::Streams::default(), listeners: listeners::Listeners::default(), @@ -131,6 +138,19 @@ impl Node { }) } + /// A link, down and unaddressed, ranked against the others by `rank`: its client begins here, + /// and its first DISCOVER is not sent: an exchange starts when the link comes up + /// ([`Self::link`]). The machine's name, once answered, is served on it too. + pub fn add_link(&mut self, now: Instant, mac: IndividualMac, rank: Rank, draw: impl FnMut() -> u32) -> IfIndex { + let iface = self.stack.add_link(now, mac, rank); + let (client, _unsent) = Client::start(now, mac, self.host_name.clone(), draw); + self.clients.push(LinkClient { link: iface, mac, client, log: toyos_dhcp::RefusalLog::default() }); + if let Some(name) = &mut self.name { + name.add_link(now, &mut self.stack, iface); + } + iface + } + pub fn shard(&self) -> &Shard { self.stack.shard() } @@ -139,10 +159,20 @@ impl Node { &self.counters } - /// The lease in use: its address verified, its router installed, its resolvers the ones to - /// ask. `None` from the moment any of them goes. - pub fn lease(&self) -> Option<&Lease> { - self.stack.lease() + /// Every link, in the order added. + pub fn links(&self) -> impl Iterator + '_ { + self.stack.links() + } + + /// `link`'s lease in use: its address verified, its router installed. `None` from the moment + /// either goes. + pub fn lease(&self, link: IfIndex) -> Option<&Lease> { + self.stack.lease(link) + } + + /// The link whose lease's resolvers lookups ask. + pub fn asked(&self) -> Option { + self.stack.asked() } /// Log lines since the last call. @@ -150,11 +180,18 @@ impl Node { self.events.drain(..) } - // ---- the device ---- + fn client(&mut self, link: IfIndex) -> &mut Client { + match self.clients.iter_mut().find(|c| c.link == link) { + Some(c) => &mut c.client, + None => unreachable!("{link:?} is no link of the node's"), + } + } + + // ---- the devices ---- - /// The frames the device received since the last call, one by one and at most `budget`: - /// `next` hands its next frame to the sink it is given and answers `false` once it had none. - /// One pass over the streams follows the last frame, so what the batch brought reaches each + /// The frames `link` received since the last call, one by one and at most `budget`: `next` + /// hands its next frame to the sink it is given and answers `false` once it had none. One + /// pass over the streams follows the last frame, so what the batch brought reaches each /// client in one write and not one a frame. `true` is a batch that stopped at the budget, and /// another is owed without waiting for the device to say so. /// @@ -162,14 +199,14 @@ impl Node { /// batch that waited for it to empty would keep the deadlines, the transmit opportunity and /// every stream's pass waiting with it, while a bound below what one interrupt's wait can /// bring would owe a second pass for every long wait. - pub fn receive(&mut self, now: Instant, budget: usize, mut next: impl FnMut(&mut dyn FnMut(&[u8])) -> bool, mut draw: impl FnMut() -> u32) -> bool { + pub fn receive(&mut self, now: Instant, link: IfIndex, budget: usize, mut next: impl FnMut(&mut dyn FnMut(&[u8])) -> bool, mut draw: impl FnMut() -> u32) -> bool { let mut taken = 0usize; let owed = loop { if taken == budget { break true; } if !next(&mut |frame| { - self.stack.receive(now, frame); + self.stack.receive(now, link, frame); self.settle(now, &mut draw); }) { break false; @@ -180,17 +217,18 @@ impl Node { owed } - /// A transmit opportunity with room for `credit` frames, each handed to `sink` as it is built. - /// Returns how many left. A lookup's query that found no way out as its turn came is read - /// here, and the query asked in its place leaves in this opportunity while credit lasts. - pub fn transmit(&mut self, now: Instant, credit: usize, mut sink: impl FnMut(&[u8]), mut draw: impl FnMut() -> u32) -> usize { + /// A transmit opportunity with `credit`'s room on each link, each frame handed to `sink` as + /// it is built, with the link it leaves by. Returns how many left. A lookup's query that found + /// no way out as its turn came is read here, and the query asked in its place leaves in this + /// opportunity while credit lasts. + pub fn transmit(&mut self, now: Instant, mut credit: impl Credit, mut sink: impl FnMut(IfIndex, &[u8]), mut draw: impl FnMut() -> u32) -> usize { let mut sent = 0usize; loop { - sent = sent.saturating_add(self.stack.transmit(now, credit.saturating_sub(sent), &mut sink)); + sent = sent.saturating_add(self.stack.transmit(now, &mut credit, &mut sink)); // A datagram [ip] refused as it left is a line of this opportunity. self.log(now); let asked = self.resolver.pass(now, &mut self.stack, &mut self.counters, &mut draw); - if !asked || sent >= credit { + if !asked || !self.stack.links().any(|link| credit.room(link)) { break; } } @@ -198,15 +236,14 @@ impl Node { sent } - /// The link came up or went down; the caller reports a change, not a state. Down, a held - /// lease stays and the machine's name has no link; up, the client verifies a lease it holds - /// and otherwise starts over, and the name of a lease that stayed is probed for again. - pub fn link(&mut self, now: Instant, up: bool, mut draw: impl FnMut() -> u32) { - self.stack.link(now, up); - self.up = up; + /// `link` came up or went down; the caller reports a change, not a state. Down, a held lease + /// stays and the machine's name has no link there; up, the link's client verifies a lease it + /// holds and otherwise starts over, and the name of a lease that stayed is probed for again. + pub fn link(&mut self, now: Instant, link: IfIndex, up: bool, mut draw: impl FnMut() -> u32) { + self.stack.link(now, link, up); if up { - let out = self.client.link_up(now, &mut draw); - self.carry_out(now, out, None, &mut draw); + let out = self.client(link).link_up(now, &mut draw); + self.carry_out(now, link, out, None, &mut draw); } self.settle(now, &mut draw); } @@ -215,65 +252,82 @@ impl Node { pub fn next_deadline(&self) -> Option { let name = self.name.as_ref().and_then(name::Name::next_deadline); - self.stack.next_deadline().into_iter().chain(self.client.next_deadline()).chain(name).chain(self.resolver.next_deadline()).chain(self.streams.next_deadline()).min() + let clients = self.clients.iter().filter_map(|c| c.client.next_deadline()); + self.stack.next_deadline().into_iter().chain(clients).chain(name).chain(self.resolver.next_deadline()).chain(self.streams.next_deadline()).min() } /// Every deadline at or before `now`; the frames they make due wait for [`Self::transmit`]. pub fn fire(&mut self, now: Instant, mut draw: impl FnMut() -> u32) { self.stack.fire(now); self.settle(now, &mut draw); - let out = self.client.timer(now, &mut draw); - self.carry_out(now, out, None, &mut draw); + for at in 0..self.clients.len() { + let Some(c) = self.clients.get_mut(at) else { continue }; + let link = c.link; + let out = c.client.timer(now, &mut draw); + self.carry_out(now, link, out, None, &mut draw); + } self.settle(now, &mut draw); self.pass(now, false); } - /// Hands the client what the shard reported and what reached its socket, and carries out - /// what it answers, until neither has more; then the name is served and the lookups are - /// carried on, against the lease as that left it. + /// Hands each link's client what the shard reported and what reached its socket, and carries + /// out what it answers, until neither has more; then the name is served and the lookups are + /// carried on, against the leases as that left them. fn settle(&mut self, now: Instant, draw: &mut impl FnMut() -> u32) { loop { self.log(now); - let (out, verified) = if let Some(report) = self.stack.report() { + let (link, out, verified) = if let Some((link, report)) = self.stack.report() { + let client = self.client(link); match report { - Report::Verified(verified) => (self.client.verified(now, &mut *draw), Some(verified)), - Report::Conflict(mac) => (self.client.conflict(now, mac, &mut *draw), None), - Report::NotVerified => (self.client.not_verified(now, &mut *draw), None), + Report::Verified(verified) => (link, client.verified(now, &mut *draw), Some(verified)), + Report::Conflict(mac) => (link, client.conflict(now, mac, &mut *draw), None), + Report::NotVerified => (link, client.not_verified(now, &mut *draw), None), } - } else if let Some((payload, from)) = self.stack.recv(&mut self.datagram) { - (self.client.receive(now, payload, from, &mut *draw), None) + } else if let Some((payload, from, link)) = self.stack.recv(&mut self.datagram) { + // On a LAN two of the links share, a reply for one link's client reaches the + // other's too: it is that link's, and this one's client never hears it. + let other = toyos_dhcp::addressed_to(payload) + .is_some_and(|to| self.clients.iter().any(|c| c.link != link && c.mac.get() == to)); + if other { + self.counters.add(Counter::DhcpOtherLink, 1); + continue; + } + let Some(c) = self.clients.iter_mut().find(|c| c.link == link) else { unreachable!("[ip] delivered from no link of the node's") }; + (link, c.client.receive(now, payload, from, &mut *draw), None) } else { break; }; - self.carry_out(now, out, verified, draw); + self.carry_out(now, link, out, verified, draw); } self.serve_name(now, draw); self.resolver.pass(now, &mut self.stack, &mut self.counters, draw); } - /// The stack's and the client's log lines so far, into [`Self::drain_events`]: the one way a - /// line leaves either. + /// The stack's and the clients' log lines so far, into [`Self::drain_events`]: the one way a + /// line leaves any of them. fn log(&mut self, now: Instant) { let events = &mut self.events; self.stack.refusals(|refusal, suppressed| events.push(Event::Stack { refusal, suppressed })); - for refusal in self.client.drain_refusals() { - if let Some(suppressed) = self.dhcp_log.admit(now, refusal.rule) { - events.push(Event::Dhcp { refusal, suppressed }); + for c in &mut self.clients { + for refusal in c.client.drain_refusals() { + if let Some(suppressed) = c.log.admit(now, refusal.rule) { + events.push(Event::Dhcp { link: c.link, refusal, suppressed }); + } } } } - /// What one call of the client's asked for: the lease first, so a message leaves from the + /// What one call of `link`'s client asked for: the lease first, so a message leaves from the /// state its call left behind. - fn carry_out(&mut self, now: Instant, out: Output, verified: Option, draw: &mut impl FnMut() -> u32) { + fn carry_out(&mut self, now: Instant, link: IfIndex, out: Output, verified: Option, draw: &mut impl FnMut() -> u32) { use toyos_dhcp::Config::{Configured, Deconfigured, Extended, Reconfigured}; let installed = match (out.config, verified) { - (Some(Configured(lease)), Some(verified)) => Some(Ok(self.stack.hold(now, verified, lease))), + (Some(Configured(lease)), Some(verified)) => Some(Ok(self.stack.hold(now, link, verified, lease))), (Some(Configured(lease)), None) => unreachable!("the client configured {} unverified", lease.address), - (Some(Extended(lease) | Reconfigured(lease)), _) => Some(self.stack.renew(now, lease)), + (Some(Extended(lease) | Reconfigured(lease)), _) => Some(self.stack.renew(now, link, lease)), // Verified, and the client has no use for it: its lease ran out under the probe. (Some(Deconfigured(_)), _) | (None, Some(_)) => { - self.stack.release(now); + self.stack.release(now, link); None } (None, None) => None, @@ -286,19 +340,19 @@ impl Node { let mut transmit = out.transmit; match out.address { Some(AddressRequest::Probe { address, prefix_len }) => { - if self.stack.probe(now, address, prefix_len).is_err() { + if self.stack.probe(now, link, address, prefix_len).is_err() { self.counters.add(Counter::AddressRefused, 1); // An address [ip] will not probe is one the client never hears about again: // its exchange starts over. - transmit = self.client.not_verified(now, &mut *draw).transmit; + transmit = self.client(link).not_verified(now, &mut *draw).transmit; } } // Only of a link reported up twice: one that drops under a probe is `NotVerified` first. - Some(AddressRequest::Cancel(_)) => self.stack.release(now), + Some(AddressRequest::Cancel(_)) => self.stack.release(now, link), None => {} } if let Some(message) = transmit { - if self.stack.send(now, &message).is_err() { + if self.stack.send(now, link, &message).is_err() { self.counters.add(Counter::DhcpUnsent, 1); } } diff --git a/system/netstack/node/src/name.rs b/system/netstack/node/src/name.rs index 9b06bd50a47..7c00e6c838d 100644 --- a/system/netstack/node/src/name.rs +++ b/system/netstack/node/src/name.rs @@ -1,9 +1,19 @@ -//! This machine's name on its link: `.local` answers with the held lease's address -//! (`toyos-mdns`, RFC 6762). Every decision about a message, and every byte of a probe or an -//! answer, is `toyos_mdns::Responder`'s; this is its socket on the group's port, its membership -//! of the group, its clock and its draws. +//! This machine's name on each of its links: `.local` answers with that link's held +//! lease's address (`toyos-mdns`, RFC 6762). Every decision about a message, and every byte of a +//! probe or an answer, is one link's `toyos_mdns::Responder`'s; this is their one socket on the +//! group's port, their membership of the group on each link, their clock and their draws. //! -//! - The record is the held lease's on a link that is up: its address and prefix are what the +//! - **One name, a responder per link.** A message is its arrival link's responder's alone, and +//! what a responder sends leaves by its link alone. A name lost on one link is that link's: the +//! responder there probes for the same name again later (§8.1), and the others keep it. +//! - **A message from one of this machine's own addresses never reaches a responder**: [ip] +//! refuses a datagram whose source any of its interfaces holds (`ip.own-source`), so on one LAN +//! two links share, as a wired and a wireless link bridged (§14), this machine hears its own +//! messages from neither and never conflicts with itself. The one exception is a message that +//! crosses the bridge as its sending link loses the address it came from: [ip] no longer holds +//! that source, a millisecond's race, and the other link's responder may read it as another +//! host's. +//! - The record is the link's held lease's on a link that is up: its address and prefix are what the //! responder is told, and it is told it has no link from the moment the lease goes or the link //! does. Nothing is probed for, announced or answered for an address still under probe, or on a //! link that is down. @@ -24,11 +34,13 @@ //! source for one on this link, which is the lease's prefix or 169.254/16, and [udp] accepted it //! as one. +use alloc::collections::BTreeMap; use alloc::vec; use alloc::vec::Vec; use core::net::Ipv4Addr; use toyos_mdns::{Host, Link, Responder, Source, To, GROUP, PORT}; +use toyos_net_shard::IfIndex; use toyos_net_udp::SocketId; use toyos_net_wire::ipv4::{MulticastAddr, Ttl}; use toyos_net_wire::{Instant, Port}; @@ -37,8 +49,10 @@ use crate::lease::Stack; use crate::{Event, Node}; pub(crate) struct Name { + host: Host<'static>, socket: SocketId, - record: Responder<'static>, + /// Each link's record, by the link. + records: BTreeMap>, /// Room for the largest datagram [udp] delivers. message: Vec, } @@ -48,78 +62,93 @@ pub(crate) fn millis(now: Instant) -> u64 { u64::try_from(now.since(Instant::from_millis(0)).as_millis()).unwrap_or(u64::MAX) } -/// Queues one message of the responder's. Returns how many [udp] refused: one or none. -fn send(stack: &mut Stack, now: Instant, socket: SocketId, to: Ipv4Addr, port: u16, message: &[u8]) -> u64 { - u64::from(stack.send_to(now, socket, to, port, message).is_err()) +fn group() -> MulticastAddr { + let Some(group) = MulticastAddr::new(Ipv4Addr::from(GROUP)) else { unreachable!("RFC 6762 §3's group is a group") }; + group } impl Name { - /// Binds the group's port, sets the TTL of what leaves it and joins the group. Refused, - /// nothing was joined. + /// Binds the group's port and sets the TTL of what leaves it, then joins the group on every + /// link. Refused, nothing was joined. fn new(now: Instant, stack: &mut Stack, host: Host<'static>) -> Result { - let (Ok(ttl), Some(group)) = (Ttl::new(u8::MAX), MulticastAddr::new(Ipv4Addr::from(GROUP))) else { - unreachable!("255 is a TTL and RFC 6762 §3's group is a group") - }; + let Ok(ttl) = Ttl::new(u8::MAX) else { unreachable!("255 is a TTL") }; let (socket, _) = stack.bind(Ipv4Addr::UNSPECIFIED, Port::new(PORT), || 0)?; stack.set_ttl(socket, ttl, ttl)?; - stack.join(now, group); - Ok(Self { socket, record: Responder::new(host), message: vec![0; toyos_net_udp::limits::MAX_PAYLOAD] }) + let mut name = Self { host, socket, records: BTreeMap::new(), message: vec![0; toyos_net_udp::limits::MAX_PAYLOAD] }; + let links: Vec = stack.links().collect(); + for link in links { + name.add_link(now, stack, link); + } + Ok(name) + } + + /// Joins the group on `link`, whose record begins with no address. + pub(crate) fn add_link(&mut self, now: Instant, stack: &mut Stack, link: IfIndex) { + stack.join(now, link, group()); + self.records.insert(link, Responder::new(self.host)); } pub(crate) fn next_deadline(&self) -> Option { - self.record.owed_at().map(Instant::from_millis) + self.records.values().filter_map(Responder::owed_at).min().map(Instant::from_millis) } - /// Tells the responder what lease is held and whether the link is `up`, hands it every - /// message that arrived and sends what each is answered, then sends what the name is owed - /// now. What became of the name goes to `events`. Returns how many messages [udp] refused. - fn pass(&mut self, now: Instant, stack: &mut Stack, up: bool, events: &mut Vec, draw: &mut impl FnMut() -> u32) -> u64 { - let link = stack.lease().filter(|_| up).map(|lease| Link { addr: lease.address.octets(), prefix: lease.prefix_len }); + /// Tells each link's responder what lease is held there and whether the link is up, hands + /// each message that arrived to its link's responder and sends what it is answered by that + /// link, then sends what each link's name is owed now. What became of a name goes to + /// `events`. Returns how many messages [udp] refused. + fn pass(&mut self, now: Instant, stack: &mut Stack, events: &mut Vec, draw: &mut impl FnMut() -> u32) -> u64 { let (ms, group) = (millis(now), Ipv4Addr::from(GROUP)); let mut unsent = 0u64; - self.record.on(link, ms); + for (&link, record) in &mut self.records { + let on = stack.lease(link).filter(|_| stack.up(link)).map(|lease| Link { addr: lease.address.octets(), prefix: lease.prefix_len }); + record.on(on, ms); + } + let socket = self.socket; + let send = |stack: &mut Stack, link: IfIndex, to: Ipv4Addr, port: u16, message: &[u8]| u64::from(stack.send_via(now, socket, link, to, port, message).is_err()); loop { let received = match stack.recv_from(self.socket, &mut self.message) { Ok(Some(received)) => received, Ok(None) => break, - Err(_) => unreachable!("the responder's socket is never closed and never connected"), + Err(_) => unreachable!("the responders' socket is never closed and never connected"), }; - let Some(message) = self.message.get(..received.len) else { continue }; + let (Some(message), Some(record)) = (self.message.get(..received.len), self.records.get_mut(&received.iface)) else { continue }; // Port 0 is no port to answer at: [udp] refuses the answer. let port = received.source_port.map_or(0, Port::get); - let (answer, event) = self.record.heard(message, Source { addr: received.source.octets(), port }, ms); - events.extend(event.map(Event::Name)); + let (answer, event) = record.heard(message, Source { addr: received.source.octets(), port }, ms); + events.extend(event.map(|event| Event::Name { link: received.iface, event })); let Some(answer) = answer else { continue }; let (to, port) = match answer.to { To::Group => (group, PORT), To::Asker => (received.source, port), }; - unsent = unsent.saturating_add(send(stack, now, self.socket, to, port, &answer.bytes)); + unsent = unsent.saturating_add(send(stack, received.iface, to, port, &answer.bytes)); } - let (owed, event) = self.record.owed(ms, draw); - events.extend(event.map(Event::Name)); - if let Some(owed) = owed { - unsent = unsent.saturating_add(send(stack, now, self.socket, group, PORT, &owed)); + for (&link, record) in &mut self.records { + let (owed, event) = record.owed(ms, &mut *draw); + events.extend(event.map(|event| Event::Name { link, event })); + if let Some(owed) = owed { + unsent = unsent.saturating_add(send(stack, link, group, PORT, &owed)); + } } unsent } } impl Node { - /// Answers for `.local` from here on, once it is claimed: the node joins the multicast - /// DNS group and holds its port, and a lease already held on a link that is up is probed on - /// by this call, which then spends one draw. Refused when the port is taken, by a client's - /// socket or by an earlier call. + /// Answers for `.local` from here on, on every link, once it is claimed there: the node + /// joins the multicast DNS group on each and holds its port, and a lease already held on a + /// link that is up is probed on by this call, which then spends one draw for each. Refused + /// when the port is taken, by a client's socket or by an earlier call. pub fn answer_as(&mut self, now: Instant, host: Host<'static>, mut draw: impl FnMut() -> u32) -> Result<(), toyos_net_udp::Error> { self.name = Some(Name::new(now, &mut self.stack, host)?); self.serve_name(now, &mut draw); Ok(()) } - /// The name's pass, if it has been started, and the count of what [udp] refused it. + /// The names' pass, if they have been started, and the count of what [udp] refused them. pub(crate) fn serve_name(&mut self, now: Instant, draw: &mut impl FnMut() -> u32) { if let Some(name) = &mut self.name { - let unsent = name.pass(now, &mut self.stack, self.up, &mut self.events, draw); + let unsent = name.pass(now, &mut self.stack, &mut self.events, draw); self.counters.add(crate::Counter::NameUnsent, unsent); } } diff --git a/system/netstack/node/src/resolve.rs b/system/netstack/node/src/resolve.rs index 917fa53147c..f3fb329cee8 100644 --- a/system/netstack/node/src/resolve.rs +++ b/system/netstack/node/src/resolve.rs @@ -207,7 +207,7 @@ impl Resolver { if let Some(kept) = stack.answers().answer(view, &name, millis(now)) { return Ok(Resolving::Kept(kept.map_err(Ended::Failed))); } - let resolvers = stack.lease().map(|lease| lease.dns.clone()).unwrap_or_default(); + let resolvers = stack.asked_lease().map(|lease| lease.dns.clone()).unwrap_or_default(); if resolvers.is_empty() { return Err(NotStarted::NotConnected); } @@ -242,7 +242,7 @@ impl Resolver { let ms = millis(now); let mut queued = false; lookups.retain_mut(|asking| { - let named = stack.lease().is_some_and(|lease| lease.dns == asking.resolvers); + let named = stack.asked_lease().is_some_and(|lease| lease.dns == asking.resolvers); let mut done = if named { None } else { Some(Err(Ended::LeaseChanged)) }; while done.is_none() { let Some((query, heard)) = waiting(&asking.queries, stack, reply.as_mut_slice()) else { break }; diff --git a/system/netstack/node/src/streams.rs b/system/netstack/node/src/streams.rs index 067ec99ac0c..f7e3e88d4d2 100644 --- a/system/netstack/node/src/streams.rs +++ b/system/netstack/node/src/streams.rs @@ -29,7 +29,7 @@ use alloc::collections::{BTreeMap, VecDeque}; use core::sync::atomic::Ordering; use core::time::Duration; -use toyos::net::stream::{Server, Words, LAYOUT, NODELAY, RECEIVE_BYTES, RESET, TIMED_OUT, UNREACHABLE}; +use toyos::net::stream::{Server, Words, ADDRESS_GONE, LAYOUT, NODELAY, RECEIVE_BYTES, RESET, TIMED_OUT, UNREACHABLE}; use toyos_net_shard::ConnectError; use toyos_net_tcp::{ConnId, Endpoint, Error, Failure, Lent, Options, State, Status}; use toyos_net_wire::{Instant, Port}; @@ -92,6 +92,7 @@ const fn code(failure: Failure) -> u32 { Failure::TimedOut => TIMED_OUT, Failure::Unreachable(_) | Failure::Prohibited => UNREACHABLE, Failure::Refused | Failure::Reset => RESET, + Failure::AddressGone => ADDRESS_GONE, } } diff --git a/system/netstack/node/tests/common/mod.rs b/system/netstack/node/tests/common/mod.rs index 862d1bbd0d5..36f54fad9dd 100644 --- a/system/netstack/node/tests/common/mod.rs +++ b/system/netstack/node/tests/common/mod.rs @@ -18,7 +18,7 @@ use std::time::Duration; use etherparse::{ArpOperation, Icmpv4Type, LinkSlice, NetSlice, SlicedPacket, TransportSlice}; use toyos_dhcp::HostName; -use toyos_net_node::{Bell, Node}; +use toyos_net_node::{Bell, IfIndex, Node, Rank}; use toyos_net_shard::{Config, Secrets}; use toyos_net_wire::ethernet::{IndividualMac, MacAddr}; use toyos_net_wire::Instant; @@ -329,6 +329,8 @@ pub fn batch<'a>(frames: &'a [&'a [u8]]) -> impl FnMut(&mut Sink<'_>) -> bool + pub struct Wire { pub node: Node, + /// The node's one link. + pub link: IfIndex, mac: [u8; 6], pub now: Instant, /// Every frame the node emitted, in order. @@ -358,11 +360,12 @@ impl Wire { tcp: toyos_net_tcp::Secrets { isn: [3; 16], timestamp: [4; 16], port_offset: [5; 16], port_index: [6; 16], port_table: [0; 16] }, }; let now = Instant::from_millis(3_600_000); - let config = Config { mac: IndividualMac::new(MacAddr(mac)).unwrap(), receive_buffer: 65_535, send_buffer: 65_535, secrets }; + let config = Config { receive_buffer: 65_535, send_buffer: 65_535, secrets }; let mut draws = 0x5a00_0000; - let mut node = Node::new(now, config, HostName::new("toyos"), draw(&mut draws)).unwrap(); + let mut node = Node::new(now, config, HostName::new("toyos")).unwrap(); + let link = node.add_link(now, IndividualMac::new(MacAddr(mac)).unwrap(), Rank(0), draw(&mut draws)); node.set_places(PLACES); - Self { node, mac, now, sent: Vec::new(), draws } + Self { node, link, mac, now, sent: Vec::new(), draws } } /// Offers the node all the credit it wants until it sends nothing more, the router answering @@ -371,13 +374,13 @@ impl Wire { loop { let from = self.sent.len(); let (sent, mac) = (&mut self.sent, self.mac); - self.node.transmit(self.now, usize::MAX, |frame| sent.push(outside(frame, mac)), draw(&mut self.draws)); + self.node.transmit(self.now, usize::MAX, |_, frame| sent.push(outside(frame, mac)), draw(&mut self.draws)); if self.sent.len() == from { return; } let asked = self.sent[from..].iter().filter(|seen| matches!(seen, Seen::Arp { request: true, sender, target, .. } if *target == R && *sender == A)).count(); for _ in 0..asked { - self.node.receive(self.now, BUDGET, batch(&[&arp(MAC, false, MAC_R, R, A)]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[&arp(MAC, false, MAC_R, R, A)]), draw(&mut self.draws)); } } } @@ -388,12 +391,12 @@ impl Wire { } pub fn deliver(&mut self, frame: &[u8]) { - self.node.receive(self.now, BUDGET, batch(&[frame]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[frame]), draw(&mut self.draws)); self.pump(); } pub fn link(&mut self, up: bool) { - self.node.link(self.now, up, draw(&mut self.draws)); + self.node.link(self.now, self.link, up, draw(&mut self.draws)); self.pump(); } @@ -434,13 +437,11 @@ impl Wire { /// The interface's address `address`, in [ip]'s words. pub fn address(&self, address: Ipv4Addr) -> Option { - let shard = self.node.shard(); - shard.ip().address(shard.iface(), address) + self.node.shard().ip().address(self.link, address) } pub fn gateways(&self) -> Vec { - let shard = self.node.shard(); - shard.ip().gateways(shard.iface()).expect("the interface").to_vec() + self.node.shard().ip().gateways(self.link).expect("the interface").to_vec() } /// Whether the node answers an ARP request for `address` from the router. @@ -465,7 +466,7 @@ impl Wire { /// [`Self::acknowledged`], and conflict detection run to its end: the lease is held. pub fn leased(terms: &[(u8, Vec)]) -> Self { let mut wire = Self::acknowledged(terms); - assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease().is_some()), "the lease is held"); + assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease(wire.link).is_some()), "the lease is held"); wire } } diff --git a/system/netstack/node/tests/datagram.rs b/system/netstack/node/tests/datagram.rs index d61de97c63f..ae37485013b 100644 --- a/system/netstack/node/tests/datagram.rs +++ b/system/netstack/node/tests/datagram.rs @@ -144,7 +144,7 @@ fn each_refusal_is_answered_in_the_pipes_word_for_it() { lan.pump(); lan.node.udp_send_to(lan.now, any, B, 7, b"later").expect("the same call, once the queue has left"); - assert!(lan.run_until(Duration::from_secs(700), |lan| lan.node.lease().is_none()), "the lease runs out"); + assert!(lan.run_until(Duration::from_secs(700), |lan| lan.node.lease(lan.link).is_none()), "the lease runs out"); refused_send(&mut lan, any, B, 7, b"late", Refused::NotConnected, Counter::NoSourceAddress); refused_send(&mut lan, held, B, 7, b"late", Refused::NotConnected, Counter::SourceAddressNotAssigned); @@ -218,13 +218,13 @@ fn a_datagram_accepted_for_a_host_is_no_broadcast_after_the_server_renews_a_narr lan.node.udp_send_to(lan.now, id, edge, 5_001, b"hi").expect("one host of the /24"); let narrower = [(54, R.octets().to_vec()), (51, 600u32.to_be_bytes().to_vec()), (1, vec![255, 255, 255, 128]), (6, DNS.octets().to_vec())]; lan.deliver(&from_server(MAC, A, &message_of(ACK, renewal, &narrower))); - assert_eq!(lan.node.lease().map(|lease| lease.prefix_len), Some(25), "the server's prefix is the lease's"); + assert_eq!(lan.node.lease(lan.link).map(|lease| lease.prefix_len), Some(25), "the server's prefix is the lease's"); assert_eq!(lan.sent[frames..], [], "nothing left for it: no broadcast, and no request for a link address"); let rule = toyos_net_ip::Counter::IpBroadcastNotPermitted; let shard = lan.node.shard(); assert_eq!(shard.ip().counters().get(rule), 1); - let refusal = toyos_net_ip::Refusal { rule, iface: shard.iface(), peer: toyos_net_ip::Peer::Ip(edge) }; + let refusal = toyos_net_ip::Refusal { rule, iface: lan.link, peer: toyos_net_ip::Peer::Ip(edge) }; let logged: Vec = lan.node.drain_events().collect(); assert_eq!(logged, [Event::Stack { refusal: Refusal::Ip(refusal), suppressed: 0 }]); } diff --git a/system/netstack/node/tests/lan/mod.rs b/system/netstack/node/tests/lan/mod.rs index de4ec988d58..bb9e52def2d 100644 --- a/system/netstack/node/tests/lan/mod.rs +++ b/system/netstack/node/tests/lan/mod.rs @@ -117,6 +117,7 @@ pub fn udp(to: [u8; 6], from: [u8; 6], source: (Ipv4Addr, u16), destination: (Ip pub struct Lan { pub node: Node, + pub link: toyos_net_node::IfIndex, pub now: Instant, /// Every frame the node emitted, in order, with the time it left. pub sent: Vec<(Instant, Seen)>, @@ -134,14 +135,14 @@ fn draw(draws: &mut u32) -> impl FnMut() -> u32 + '_ { impl Lan { /// `common`'s node, an hour into the clock, its link down. pub fn new() -> Self { - let Wire { node, now, .. } = Wire::new(); - Self { node, now, sent: Vec::new(), draws: 0x6b00_0000 } + let Wire { node, link, now, .. } = Wire::new(); + Self { node, link, now, sent: Vec::new(), draws: 0x6b00_0000 } } /// One transmit opportunity with all the credit the node wants. Returns how many frames left. pub fn opportunity(&mut self) -> usize { let (sent, now) = (&mut self.sent, self.now); - self.node.transmit(now, usize::MAX, |frame| sent.push((now, outside(frame))), draw(&mut self.draws)) + self.node.transmit(now, usize::MAX, |_, frame| sent.push((now, outside(frame))), draw(&mut self.draws)) } /// Opportunities until the node sends nothing more, the router and B answering each ARP @@ -161,18 +162,18 @@ impl Lan { .collect(); for target in asked { let mac = if target == R { MAC_R } else { MAC_B }; - self.node.receive(self.now, BUDGET, batch(&[&arp(MAC, false, mac, target, A)]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[&arp(MAC, false, mac, target, A)]), draw(&mut self.draws)); } } } pub fn deliver(&mut self, frame: &[u8]) { - self.node.receive(self.now, BUDGET, batch(&[frame]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[frame]), draw(&mut self.draws)); self.pump(); } pub fn link(&mut self, up: bool) { - self.node.link(self.now, up, draw(&mut self.draws)); + self.node.link(self.now, self.link, up, draw(&mut self.draws)); self.pump(); } @@ -208,13 +209,13 @@ impl Lan { let id = xid(&discover); self.deliver(&from_server(MAC, A, &message_of(OFFER, id, terms))); self.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, id, terms))); - assert_eq!(self.node.lease(), None, "acknowledged is not held"); + assert_eq!(self.node.lease(self.link), None, "acknowledged is not held"); } /// [`Self::acknowledge`], and conflict detection run to its end: the lease is held. pub fn lease_on(&mut self, terms: &[(u8, Vec)]) { self.acknowledge(terms); - assert!(self.run_until(Duration::from_secs(10), |lan| lan.node.lease().is_some()), "the lease is held"); + assert!(self.run_until(Duration::from_secs(10), |lan| lan.node.lease(lan.link).is_some()), "the lease is held"); } /// A lease of `A`/24 for `seconds`, through the router. diff --git a/system/netstack/node/tests/lease.rs b/system/netstack/node/tests/lease.rs index 460a578bb3f..b35f99f7b3e 100644 --- a/system/netstack/node/tests/lease.rs +++ b/system/netstack/node/tests/lease.rs @@ -56,15 +56,15 @@ fn the_lease_end_to_end() { let acknowledged = wire.sent.len(); wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, id, &terms))); - assert_eq!((wire.node.lease(), wire.address(A)), (None, Some(AddrState::Tentative))); + assert_eq!((wire.node.lease(wire.link), wire.address(A)), (None, Some(AddrState::Tentative))); - assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease().is_some()), "the lease is held"); + assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease(wire.link).is_some()), "the lease is held"); let detection = &wire.sent[acknowledged..]; assert_eq!(detection.len(), 4, "three probes and an announcement: {detection:?}"); assert!(detection[..3].iter().all(|frame| frame.is_probe(A)), "{detection:?}"); assert!(detection[3].is_announcement(A), "{detection:?}"); - let lease = wire.node.lease().unwrap(); + let lease = wire.node.lease(wire.link).unwrap(); assert_eq!((lease.address, lease.prefix_len, lease.router, lease.dns.as_slice(), lease.server), (A, 24, Some(R), &[DNS][..], R)); assert_eq!(wire.address(A), Some(AddrState::Announcing)); assert_eq!(wire.gateways(), [R]); @@ -81,9 +81,9 @@ fn the_lease_end_to_end() { #[test] fn no_frame_is_from_the_address_before_it_is_verified() { let mut wire = Wire::acknowledged(&terms(HOUR, Some(R))); - assert_eq!(wire.node.lease(), None, "acknowledged is not held"); + assert_eq!(wire.node.lease(wire.link), None, "acknowledged is not held"); for _ in 0..16 { - if wire.node.lease().is_some() { + if wire.node.lease(wire.link).is_some() { break; } assert_eq!(wire.address(A), Some(AddrState::Tentative)); @@ -92,7 +92,7 @@ fn no_frame_is_from_the_address_before_it_is_verified() { let at = wire.node.next_deadline().expect("conflict detection is running"); wire.fire(at); } - assert!(wire.node.lease().is_some(), "held within sixteen deadlines"); + assert!(wire.node.lease(wire.link).is_some(), "held within sixteen deadlines"); let probes = wire.sent.iter().filter(|frame| frame.is_probe(A)).count(); assert_eq!(probes, 3, "held only once the third probe has left"); assert!(wire.sent.iter().all(|frame| frame.source() != Some(A)), "{:?}", wire.sent); @@ -101,10 +101,10 @@ fn no_frame_is_from_the_address_before_it_is_verified() { #[test] fn expiry_takes_address_route_and_resolvers_in_one_step() { let mut wire = Wire::leased(&terms(600, Some(R))); - assert_eq!((wire.address(A).is_some(), wire.gateways(), wire.node.lease().map(|lease| lease.dns.clone())), (true, vec![R], Some(vec![DNS]))); + assert_eq!((wire.address(A).is_some(), wire.gateways(), wire.node.lease(wire.link).map(|lease| lease.dns.clone())), (true, vec![R], Some(vec![DNS]))); let held = wire.sent.len(); - assert!(wire.run_until(Duration::from_secs(700), |wire| wire.node.lease().is_none()), "the lease runs out"); + assert!(wire.run_until(Duration::from_secs(700), |wire| wire.node.lease(wire.link).is_none()), "the lease runs out"); assert_eq!(wire.now, after(600), "at its expiry"); assert_eq!((wire.address(A), wire.gateways()), (None, vec![])); let renewal = wire.sent[held..].iter().find(|frame| frame.dhcp().is_some()).expect("a renewal"); @@ -136,13 +136,13 @@ fn a_renewal_for_a_server_nobody_answers_for_waits_and_is_dropped_and_the_client assert!(wire.run_until(Duration::from_secs(10), |wire| dropped(wire) == 1), "the renewal is dropped"); assert_eq!((wire.now, asks(&wire)), (renewed.after(Duration::from_secs(3)), 3), "when [ip] gave the server up"); assert!(wire.sent[held..].iter().all(|seen| seen.dhcp().is_none()), "no renewal left"); - assert!(wire.node.lease().is_some(), "the lease is held, and nothing reached the client"); + assert!(wire.node.lease(wire.link).is_some(), "the lease is held, and nothing reached the client"); assert!(wire.run_until(Duration::from_secs(400), |wire| wire.sent[held..].iter().any(|seen| seen.dhcp().is_some())), "the client asks on"); assert!(dropped(&wire) > 1, "a renewal again first, on the client's clock"); let Some(Seen::Dhcp { to, source, destination, message }) = wire.sent.last() else { panic!("{:?}", wire.sent.last()) }; assert_eq!((*to, *source, *destination, option(message, 53)), (BROADCAST, A, Ipv4Addr::BROADCAST, Some(&[REQUEST][..])), "the rebinding request"); - assert!(wire.node.lease().is_some()); + assert!(wire.node.lease(wire.link).is_some()); } #[test] @@ -150,10 +150,10 @@ fn a_nak_takes_address_route_and_resolvers_in_one_step() { let mut wire = Wire::leased(&terms(600, Some(R))); let id = renewing(&mut wire); wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message(NAK, id, Ipv4Addr::UNSPECIFIED, &[(54, R.octets().to_vec())]))); - assert_eq!((wire.node.lease(), wire.address(A), wire.gateways()), (None, None, vec![])); + assert_eq!((wire.node.lease(wire.link), wire.address(A), wire.gateways()), (None, None, vec![])); assert!(matches!(wire.sent.last(), Some(Seen::Dhcp { source, .. }) if source.is_unspecified()), "{:?}", wire.sent.last()); assert_ne!(xid(wire.last(DISCOVER)), id, "a new exchange"); - let refused = Event::Dhcp { refusal: toyos_dhcp::Refusal { rule: toyos_dhcp::Counter::Nak, peer: toyos_dhcp::Peer::From(R) }, suppressed: 0 }; + let refused = Event::Dhcp { link: wire.link, refusal: toyos_dhcp::Refusal { rule: toyos_dhcp::Counter::Nak, peer: toyos_dhcp::Peer::From(R) }, suppressed: 0 }; assert!(wire.node.drain_events().any(|event| event == refused)); assert!(!wire.answers_arp_for(A)); } @@ -164,7 +164,7 @@ fn a_renewal_without_a_router_takes_the_route_and_keeps_the_address() { let id = renewing(&mut wire); let renewed = wire.now; wire.deliver(&from_server(MAC, A, &message_of(ACK, id, &terms(600, None)))); - let lease = wire.node.lease().expect("still held"); + let lease = wire.node.lease(wire.link).expect("still held"); assert_eq!((lease.address, lease.router, lease.timers.map(|timers| timers.expiry)), (A, None, Some(renewed.after(Duration::from_secs(600))))); assert_eq!((wire.address(A), wire.gateways()), (Some(AddrState::Assigned), vec![])); assert_eq!(wire.node.counters().get(Counter::RouterRefused), 0); @@ -178,10 +178,10 @@ fn a_renewed_prefix_reaches_ip_before_its_router() { let (wider, beyond) = (Ipv4Addr::new(255, 255, 254, 0), Ipv4Addr::new(192, 0, 3, 254)); let terms = [(54, R.octets().to_vec()), (51, 600u32.to_be_bytes().to_vec()), (1, wider.octets().to_vec()), (3, beyond.octets().to_vec()), (6, DNS.octets().to_vec())]; wire.deliver(&from_server(MAC, A, &message_of(ACK, id, &terms))); - let lease = wire.node.lease().expect("still held"); + let lease = wire.node.lease(wire.link).expect("still held"); assert_eq!((lease.prefix_len, lease.router), (23, Some(beyond))); let shard = wire.node.shard(); - assert_eq!(shard.ip().prefix_len(shard.iface(), A), Some(23)); + assert_eq!(shard.ip().prefix_len(wire.link, A), Some(23)); assert_eq!(wire.gateways(), [beyond]); assert_eq!(wire.node.counters().get(Counter::RouterRefused), 0); } @@ -194,7 +194,7 @@ fn a_lease_that_runs_out_under_the_probe_leaves_no_address() { assert!(wire.run_until(Duration::from_secs(10), |wire| wire.dhcp().len() == 3), "the client starts over"); wire.last(DISCOVER); assert_eq!(wire.sent.iter().filter(|frame| frame.is_probe(A)).count(), 3, "detection ran to its end"); - assert_eq!((wire.node.lease(), wire.address(A), wire.gateways()), (None, None, vec![])); + assert_eq!((wire.node.lease(wire.link), wire.address(A), wire.gateways()), (None, None, vec![])); assert!(!wire.answers_arp_for(A)); } @@ -207,12 +207,12 @@ fn a_second_conflict_takes_a_held_lease_and_declines_it() { let from = wire.sent.len(); wire.deliver(&conflict); assert_eq!(wire.sent[from..].iter().filter(|frame| frame.is_announcement(A)).count(), 1, "defended once: {:?}", &wire.sent[from..]); - assert_eq!((wire.node.lease().map(|lease| lease.address), wire.gateways()), (Some(A), vec![R])); + assert_eq!((wire.node.lease(wire.link).map(|lease| lease.address), wire.gateways()), (Some(A), vec![R])); // Inside [ip]'s defend interval (RFC 5227 §2.4 (b)). wire.fire(after(105)); wire.deliver(&conflict); - assert_eq!((wire.node.lease(), wire.address(A), wire.gateways()), (None, None, vec![])); + assert_eq!((wire.node.lease(wire.link), wire.address(A), wire.gateways()), (None, None, vec![])); let decline = wire.last(DECLINE); assert_eq!((option(decline, 50), option(decline, 54)), (Some(&A.octets()[..]), Some(&R.octets()[..]))); assert!(!wire.answers_arp_for(A)); @@ -284,7 +284,7 @@ fn losses() -> [(&'static str, Loss); 3] { wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message(NAK, id, Ipv4Addr::UNSPECIFIED, &[(54, R.octets().to_vec())]))); } fn expiry(wire: &mut Wire) { - assert!(wire.run_until(Duration::from_secs(700), |wire| wire.node.lease().is_none()), "the lease runs out"); + assert!(wire.run_until(Duration::from_secs(700), |wire| wire.node.lease(wire.link).is_none()), "the lease runs out"); } fn conflict(wire: &mut Wire) { let conflict = arp(BROADCAST, false, MAC_B, A, A); @@ -320,11 +320,11 @@ fn a_listener_stands_while_its_address_is_lost_and_answers_when_it_is_back() { wire.link(true); let id = xid(wire.last(REQUEST)); wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, id, &terms(600, Some(R))))); - assert_eq!(wire.node.lease().map(|lease| lease.address), Some(A), "{how}"); + assert_eq!(wire.node.lease(wire.link).map(|lease| lease.address), Some(A), "{how}"); stand(&mut wire, "renewed, and the link back"); lose(&mut wire); - assert_eq!((wire.node.lease(), wire.address(A)), (None, None), "{how}"); + assert_eq!((wire.node.lease(wire.link), wire.address(A)), (None, None), "{how}"); stand(&mut wire, "the address lost"); assert_eq!(syn(&mut wire, 22), [], "{how}: a SYN to an address the machine does not hold"); stand(&mut wire, "a SYN to the lost address"); @@ -334,7 +334,7 @@ fn a_listener_stands_while_its_address_is_lost_and_answers_when_it_is_back() { let id = xid(wire.last(DISCOVER)); wire.deliver(&from_server(MAC, A, &message_of(OFFER, id, &terms(600, Some(R))))); wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, id, &terms(600, Some(R))))); - assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease().is_some()), "{how}: the address is held again"); + assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease(wire.link).is_some()), "{how}: the address is held again"); stand(&mut wire, "the address back"); let answer = syn(&mut wire, 22); @@ -349,7 +349,7 @@ fn a_listener_stands_while_its_address_is_lost_and_answers_when_it_is_back() { fn a_router_ip_refuses_is_no_router() { let edge = Ipv4Addr::new(192, 0, 2, 255); let wire = Wire::leased(&terms(HOUR, Some(edge))); - assert_eq!(wire.node.lease().map(|lease| lease.router), Some(None)); + assert_eq!(wire.node.lease(wire.link).map(|lease| lease.router), Some(None)); assert!(wire.gateways().is_empty()); assert_eq!(wire.node.counters().get(Counter::RouterRefused), 1); } @@ -366,7 +366,7 @@ fn a_conflict_under_the_probe_declines_and_holds_nothing() { let Some(Seen::Dhcp { to, source, destination, message }) = wire.sent.last() else { panic!("{:?}", wire.sent.last()) }; assert_eq!((*to, *source, *destination), (BROADCAST, Ipv4Addr::UNSPECIFIED, Ipv4Addr::BROADCAST)); assert_eq!((option(message, 53), option(message, 50), option(message, 54)), (Some(&[DECLINE][..]), Some(&A.octets()[..]), Some(&R.octets()[..]))); - assert_eq!((wire.node.lease(), wire.address(A)), (None, None)); + assert_eq!((wire.node.lease(wire.link), wire.address(A)), (None, None)); let events: Vec = wire.node.drain_events().collect(); let stack = |event: &Event| matches!(event, Event::Stack { refusal: Refusal::Ip(refusal), .. } if refusal.rule == toyos_net_ip::Counter::AcdConflict); let dhcp = |event: &Event| matches!(event, Event::Dhcp { refusal, .. } if refusal.rule == toyos_dhcp::Counter::Declined); @@ -374,7 +374,7 @@ fn a_conflict_under_the_probe_declines_and_holds_nothing() { assert!(wire.run_until(Duration::from_secs(11), |wire| wire.dhcp().last().is_some_and(|(kind, _)| *kind == DISCOVER)), "discovery starts over"); assert_eq!(wire.now, conflict.after(toyos_dhcp::limits::DECLINE_BACKOFF)); - assert_eq!(wire.node.lease(), None); + assert_eq!(wire.node.lease(wire.link), None); } #[test] @@ -382,18 +382,18 @@ fn a_link_that_returns_keeps_a_held_lease_and_announces_it() { let mut wire = Wire::leased(&terms(HOUR, Some(R))); wire.fire(after(100)); wire.link(false); - assert_eq!((wire.node.lease().map(|lease| lease.address), wire.address(A).is_some(), wire.gateways()), (Some(A), true, vec![R])); + assert_eq!((wire.node.lease(wire.link).map(|lease| lease.address), wire.address(A).is_some(), wire.gateways()), (Some(A), true, vec![R])); let down = wire.sent.len(); wire.link(true); - assert_eq!(wire.node.lease().map(|lease| lease.address), Some(A)); + assert_eq!(wire.node.lease(wire.link).map(|lease| lease.address), Some(A)); assert!(wire.sent[down..].iter().any(|frame| frame.is_announcement(A)), "{:?}", &wire.sent[down..]); let request = wire.last(REQUEST); assert_eq!((ciaddr(request), option(request, 50), option(request, 54)), (Ipv4Addr::UNSPECIFIED, Some(&A.octets()[..]), None)); let id = xid(request); wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, id, &terms(HOUR, Some(R))))); - let expiry = wire.node.lease().and_then(|lease| lease.timers).map(|timers| timers.expiry); + let expiry = wire.node.lease(wire.link).and_then(|lease| lease.timers).map(|timers| timers.expiry); assert_eq!(expiry, Some(after(100 + u64::from(HOUR))), "the lease runs from the request the server acknowledged"); assert_eq!(wire.gateways(), [R]); } @@ -415,7 +415,7 @@ fn a_probe_the_link_drops_under_is_abandoned() { let mut wire = Wire::acknowledged(&terms(HOUR, Some(R))); let sent = wire.sent.len(); wire.link(false); - assert_eq!((wire.node.lease(), wire.address(A)), (None, None)); + assert_eq!((wire.node.lease(wire.link), wire.address(A)), (None, None)); assert_eq!((wire.sent.len(), wire.node.counters().get(Counter::DhcpUnsent)), (sent, 1), "its DISCOVER has no link to leave by"); wire.link(true); wire.last(DISCOVER); @@ -468,7 +468,7 @@ fn a_reply_without_the_cookie_is_refused_by_name() { bootp[236..240].fill(0); wire.deliver(&from_server(MAC, A, &bootp)); wire.last(DISCOVER); - let refused = Event::Dhcp { refusal: toyos_dhcp::Refusal { rule: toyos_dhcp::Counter::BootpReply, peer: toyos_dhcp::Peer::From(R) }, suppressed: 0 }; + let refused = Event::Dhcp { link: wire.link, refusal: toyos_dhcp::Refusal { rule: toyos_dhcp::Counter::BootpReply, peer: toyos_dhcp::Peer::From(R) }, suppressed: 0 }; assert!(wire.node.drain_events().any(|event| event == refused)); assert_eq!(wire.node.drain_events().count(), 0, "drained"); } @@ -486,11 +486,11 @@ fn a_hundred_refused_replies_are_one_line_and_a_count() { for _ in 0..100 { wire.deliver(&from_server(MAC, A, &bootp)); } - assert_eq!(wire.node.drain_events().collect::>(), [Event::Dhcp { refusal, suppressed: 0 }]); + assert_eq!(wire.node.drain_events().collect::>(), [Event::Dhcp { link: wire.link, refusal, suppressed: 0 }]); wire.now = wire.now.after(toyos_net_wire::REFUSAL_LOG_INTERVAL); wire.deliver(&from_server(MAC, A, &bootp)); - assert_eq!(wire.node.drain_events().collect::>(), [Event::Dhcp { refusal, suppressed: 99 }]); + assert_eq!(wire.node.drain_events().collect::>(), [Event::Dhcp { link: wire.link, refusal, suppressed: 99 }]); } /// The node waiting for the ACK of its REQUEST, and that ACK's frame. @@ -526,9 +526,9 @@ fn no_flipped_bit_of_the_ack_yields_another_lease() { // A flip the frame's checksums do not cover is in its link addresses, which name no // lease: taken or refused, the lease is the server's or none. let taken = wire.address(A).is_some(); - assert_eq!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease().is_some()), taken, "byte {index} bit {bit}"); + assert_eq!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease(wire.link).is_some()), taken, "byte {index} bit {bit}"); let offered = |lease: &toyos_dhcp::Lease| (lease.address, lease.prefix_len, lease.router, lease.dns.clone()) == (A, 24, Some(R), vec![DNS]); - assert!(wire.node.lease().is_none_or(offered), "byte {index} bit {bit}: {:?}", wire.node.lease()); + assert!(wire.node.lease(wire.link).is_none_or(offered), "byte {index} bit {bit}: {:?}", wire.node.lease(wire.link)); } } } diff --git a/system/netstack/node/tests/links.rs b/system/netstack/node/tests/links.rs new file mode 100644 index 00000000000..7485fbb0465 --- /dev/null +++ b/system/netstack/node/tests/links.rs @@ -0,0 +1,435 @@ +//! A node over two links, the wireless one added first and the wired one second: each leases by +//! its own client, lookups ask the wired link's resolvers and follow the links up and down, and +//! the machine's name is probed for, held and lost on each link by itself. + +use std::net::Ipv4Addr; +use std::time::Duration; + +use etherparse::{ArpOperation, LinkSlice, NetSlice, PacketBuilder, SlicedPacket, TransportSlice}; +use toyos_dhcp::HostName; +use toyos_dns::Name; +use toyos_mdns::Host; +use toyos_net_node::{Event, IfIndex, Node, Rank}; +use toyos_net_shard::{Config, Secrets}; +use toyos_net_wire::ethernet::{IndividualMac, MacAddr}; +use toyos_net_wire::Instant; + +const WIRED: Rank = Rank(0); +const WIRELESS: Rank = Rank(1); +const ROUTER_MAC: [u8; 6] = [2, 0, 0, 0, 0, 0xfe]; +const OTHER_HOST_MAC: [u8; 6] = [2, 0, 0, 0, 0, 0xee]; +const GROUP: Ipv4Addr = Ipv4Addr::new(224, 0, 0, 251); +const GROUP_MAC: [u8; 6] = [0x01, 0x00, 0x5e, 0x00, 0x00, 0xfb]; +const MDNS: u16 = 5_353; + +/// One network: the node's link address on it, the address its server leases, and its router, +/// which is its DHCP server, and its resolver. +struct Net { + mac: [u8; 6], + addr: Ipv4Addr, + router: Ipv4Addr, + dns: Ipv4Addr, +} + +const WIRED_NET: Net = Net { mac: [2, 0, 0, 0, 0, 0x01], addr: Ipv4Addr::new(192, 0, 2, 10), router: Ipv4Addr::new(192, 0, 2, 1), dns: Ipv4Addr::new(192, 0, 2, 53) }; +/// A wired and a wireless link on one LAN: one router and server, one resolver. +const LAN_WIRED: Net = Net { mac: [2, 0, 0, 0, 0, 0x01], addr: Ipv4Addr::new(192, 168, 1, 10), router: Ipv4Addr::new(192, 168, 1, 1), dns: Ipv4Addr::new(192, 168, 1, 53) }; +const LAN_WIRELESS: Net = Net { mac: [2, 0, 0, 0, 0, 0x02], addr: Ipv4Addr::new(192, 168, 1, 20), router: Ipv4Addr::new(192, 168, 1, 1), dns: Ipv4Addr::new(192, 168, 1, 53) }; +const WIRELESS_NET: Net = + Net { mac: [2, 0, 0, 0, 0, 0x02], addr: Ipv4Addr::new(198, 51, 100, 10), router: Ipv4Addr::new(198, 51, 100, 1), dns: Ipv4Addr::new(198, 51, 100, 53) }; + +/// A frame the node sent, as the outside reads it. +#[derive(Clone, Debug, PartialEq, Eq)] +enum Out { + ArpRequest { sender: Ipv4Addr, target: Ipv4Addr }, + /// A DHCP message: its type and transaction id. + Dhcp { kind: u8, xid: u32 }, + Udp { source: Ipv4Addr, destination: Ipv4Addr, port: u16, payload: Vec }, + Other, +} + +fn read(frame: &[u8]) -> Out { + let packet = SlicedPacket::from_ethernet(frame).expect("Ethernet II"); + assert!(matches!(packet.link, Some(LinkSlice::Ethernet2(_)))); + match (&packet.net, &packet.transport) { + (Some(NetSlice::Arp(arp)), None) if arp.operation() == ArpOperation::REQUEST => { + let v4 = |b: &[u8]| Ipv4Addr::new(b[0], b[1], b[2], b[3]); + Out::ArpRequest { sender: v4(arp.sender_protocol_addr()), target: v4(arp.target_protocol_addr()) } + } + (Some(NetSlice::Ipv4(_)), Some(TransportSlice::Udp(udp))) if udp.destination_port() == 67 => { + let message = udp.payload(); + let xid = u32::from_be_bytes(message[4..8].try_into().unwrap()); + let mut at = 240; + let kind = loop { + let (code, len) = (message[at], usize::from(message[at + 1])); + if code == 53 { + break message[at + 2]; + } + at += 2 + len; + }; + Out::Dhcp { kind, xid } + } + (Some(NetSlice::Ipv4(ip)), Some(TransportSlice::Udp(udp))) => { + let header = ip.header(); + Out::Udp { source: header.source_addr(), destination: header.destination_addr(), port: udp.destination_port(), payload: udp.payload().to_vec() } + } + _ => Out::Other, + } +} + +fn udp_frame(to: [u8; 6], from: [u8; 6], source: (Ipv4Addr, u16), destination: (Ipv4Addr, u16), payload: &[u8]) -> Vec { + let builder = PacketBuilder::ethernet2(from, to).ipv4(source.0.octets(), destination.0.octets(), 64).udp(source.1, destination.1); + let mut frame = Vec::with_capacity(builder.size(payload.len())); + builder.write(&mut frame, payload).unwrap(); + frame +} + +/// The router's ARP reply to the node on `net`, for `addr`, the router or the resolver. +fn arp_reply(net: &Net, addr: Ipv4Addr) -> Vec { + let mut body = vec![0, 1, 0x08, 0, 6, 4, 0, 2]; + body.extend_from_slice(&ROUTER_MAC); + body.extend_from_slice(&addr.octets()); + body.extend_from_slice(&net.mac); + body.extend_from_slice(&net.addr.octets()); + [&net.mac[..], &ROUTER_MAC, &[0x08, 0x06], &body].concat() +} + +/// `net`'s server's `kind` message for transaction `xid`, offering or acknowledging its address +/// for an hour, with its router and resolver; to the client's own addresses, or broadcast. +fn dhcp(net: &Net, kind: u8, xid: u32, broadcast: bool) -> Vec { + let mut m = vec![2, 1, 6, 0]; + m.extend_from_slice(&xid.to_be_bytes()); + m.extend_from_slice(&[0; 8]); + m.extend_from_slice(&net.addr.octets()); + m.extend_from_slice(&[0; 8]); + m.extend_from_slice(&net.mac); + m.resize(236, 0); + m.extend_from_slice(&[0x63, 0x82, 0x53, 0x63]); + let options: [(u8, Vec); 6] = [ + (53, vec![kind]), + (54, net.router.octets().to_vec()), + (51, 3_600u32.to_be_bytes().to_vec()), + (1, vec![255, 255, 255, 0]), + (3, net.router.octets().to_vec()), + (6, net.dns.octets().to_vec()), + ]; + for (code, value) in options { + m.push(code); + m.push(u8::try_from(value.len()).unwrap()); + m.extend_from_slice(&value); + } + m.push(255); + match broadcast { + false => udp_frame(net.mac, ROUTER_MAC, (net.router, 67), (net.addr, 68), &m), + true => udp_frame([0xff; 6], ROUTER_MAC, (net.router, 67), (Ipv4Addr::BROADCAST, 68), &m), + } +} + +struct Two { + node: Node, + wired: IfIndex, + wireless: IfIndex, + now: Instant, + draws: u32, + /// Every frame the node sent, with the link it left by. + sent: Vec<(IfIndex, Out)>, + /// The two links are bridged onto one LAN: a group or broadcast frame sent on one is heard + /// on the other, and a server's reply reaches both. + bridged: bool, + wired_net: &'static Net, + wireless_net: &'static Net, +} + +fn counter(draws: &mut u32) -> impl FnMut() -> u32 + '_ { + move || { + *draws += 1; + *draws + } +} + +impl Two { + fn new() -> Self { + let secrets = Secrets { + ip: [1; 16], + resets: [2; 16], + tcp: toyos_net_tcp::Secrets { isn: [3; 16], timestamp: [4; 16], port_offset: [5; 16], port_index: [6; 16], port_table: [0; 16] }, + }; + let now = Instant::from_millis(3_600_000); + let mut draws = 0x4c00_0000; + let mut node = Node::new(now, Config { receive_buffer: 65_535, send_buffer: 65_535, secrets }, HostName::new("toyos")).unwrap(); + node.set_places(8); + let wireless = node.add_link(now, IndividualMac::new(MacAddr(WIRELESS_NET.mac)).unwrap(), WIRELESS, counter(&mut draws)); + let wired = node.add_link(now, IndividualMac::new(MacAddr(WIRED_NET.mac)).unwrap(), WIRED, counter(&mut draws)); + Self { node, wired, wireless, now, draws, sent: Vec::new(), bridged: false, wired_net: &WIRED_NET, wireless_net: &WIRELESS_NET } + } + + /// Both links on one LAN behind one server. + fn bridged() -> Self { + Self { bridged: true, wired_net: &LAN_WIRED, wireless_net: &LAN_WIRELESS, ..Self::new() } + } + + fn net(&self, link: IfIndex) -> &'static Net { + if link == self.wired { self.wired_net } else { self.wireless_net } + } + + fn other(&self, link: IfIndex) -> IfIndex { + if link == self.wired { self.wireless } else { self.wired } + } + + fn deliver(&mut self, link: IfIndex, frame: &[u8]) { + let mut frames = Some(frame); + self.node.receive(self.now, link, 256, |sink| frames.take().map(&mut *sink).is_some(), counter(&mut self.draws)); + self.pump(); + } + + /// Every frame the node will send with all the credit it wants, the router answering each + /// ARP request for its own or its resolver's address, on the link it came by. + fn pump(&mut self) { + loop { + let mut out = Vec::new(); + let mut heard = Vec::new(); + let bridged = self.bridged; + self.node.transmit( + self.now, + usize::MAX, + |link, frame| { + out.push((link, read(frame))); + // A group or broadcast frame's destination has its low bit of the first byte set. + if bridged && frame[0] & 1 == 1 { + heard.push((link, frame.to_vec())); + } + }, + counter(&mut self.draws), + ); + if out.is_empty() { + return; + } + for (link, frame) in heard { + let other = self.other(link); + let mut frames = Some(frame.as_slice()); + self.node.receive(self.now, other, 256, |sink| frames.take().map(&mut *sink).is_some(), counter(&mut self.draws)); + } + let replies: Vec<(IfIndex, Vec)> = out + .iter() + .filter_map(|(link, seen)| { + let net = self.net(*link); + match seen { + Out::ArpRequest { target, .. } if *target == net.router || *target == net.dns => Some((*link, arp_reply(net, *target))), + _ => None, + } + }) + .collect(); + self.sent.extend(out); + for (link, reply) in replies { + let mut frames = Some(reply.as_slice()); + self.node.receive(self.now, link, 256, |sink| frames.take().map(&mut *sink).is_some(), counter(&mut self.draws)); + } + } + } + + fn link(&mut self, link: IfIndex, up: bool) { + self.node.link(self.now, link, up, counter(&mut self.draws)); + self.pump(); + } + + /// Fires deadline after deadline until `done`, giving up past `limit`. + fn run_until(&mut self, limit: Duration, mut done: impl FnMut(&mut Two) -> bool) -> bool { + let end = self.now.after(limit); + loop { + if done(self) { + return true; + } + match self.node.next_deadline() { + Some(at) if at <= end => { + self.now = self.now.max(at); + self.node.fire(self.now, counter(&mut self.draws)); + self.pump(); + } + _ => return false, + } + } + } + + /// The last DHCP message of `kind` that left by `link`: its transaction id. + fn last_dhcp(&self, link: IfIndex, kind: u8) -> u32 { + self.sent.iter().rev().find_map(|(at, out)| match out { + Out::Dhcp { kind: k, xid } if *at == link && *k == kind => Some(*xid), + _ => None, + }) + .expect("a DHCP message of that type on the link") + } + + /// A server's reply on `link`, which on a bridged LAN reaches the other link too. + fn reply(&mut self, link: IfIndex, frame: &[u8]) { + if self.bridged { + let other = self.other(link); + self.deliver(other, frame); + } + self.deliver(link, frame); + } + + /// `link` up, and its server's offer and acknowledgement through to a lease held. + fn lease(&mut self, link: IfIndex) { + self.link(link, true); + let net = self.net(link); + let xid = self.last_dhcp(link, 1); + // A server on a LAN two links share broadcasts its replies, so each link hears both. + let broadcast = self.bridged; + self.reply(link, &dhcp(net, 2, xid, broadcast)); + assert_eq!(self.last_dhcp(link, 3), xid, "the REQUEST"); + self.reply(link, &dhcp(net, 5, xid, broadcast)); + assert!(self.run_until(Duration::from_secs(10), |two| two.node.lease(link).is_some()), "the lease is held"); + } + + fn leased() -> Self { + Self::new().leasing() + } + + fn leasing(mut self) -> Self { + let (wired, wireless) = (self.wired, self.wireless); + self.lease(wireless); + self.lease(wired); + self + } + + /// The DNS queries that left from `from`, each with its link and resolver. + fn queries(&self, from: usize) -> Vec<(IfIndex, Ipv4Addr)> { + self.sent[from..] + .iter() + .filter_map(|(link, out)| match out { + Out::Udp { destination, port: 53, .. } => Some((*link, *destination)), + _ => None, + }) + .collect() + } + + fn resolve(&mut self, text: &str) { + let name = Name::parse(text).expect("a host name"); + self.node.resolve(self.now, b"view", name, counter(&mut self.draws)).expect("a lookup started"); + self.pump(); + } +} + +#[test] +fn each_link_leases_its_own_address_by_its_own_client() { + let two = Two::leased(); + let discovers: Vec<_> = two.sent.iter().filter(|(_, out)| matches!(out, Out::Dhcp { kind: 1, .. })).collect(); + assert!(discovers.iter().any(|(link, _)| *link == two.wired) && discovers.iter().any(|(link, _)| *link == two.wireless)); + assert_ne!(two.last_dhcp(two.wired, 1), two.last_dhcp(two.wireless, 1), "two exchanges"); + assert_eq!(two.node.lease(two.wired).map(|l| l.address), Some(WIRED_NET.addr)); + assert_eq!(two.node.lease(two.wireless).map(|l| l.address), Some(WIRELESS_NET.addr)); +} + +#[test] +fn lookups_ask_the_wired_links_resolver_fail_over_and_return() { + let mut two = Two::leased(); + let (wired, wireless) = (two.wired, two.wireless); + assert_eq!(two.node.asked(), Some(wired), "wired before wireless, though it came second"); + let from = two.sent.len(); + two.resolve("one.example"); + assert_eq!(two.queries(from), [(wired, WIRED_NET.dns)]); + + two.link(wired, false); + assert_eq!(two.node.asked(), Some(wireless), "failed over"); + let from = two.sent.len(); + two.resolve("two.example"); + assert_eq!(two.queries(from), [(wireless, WIRELESS_NET.dns)]); + + two.link(wired, true); + assert!(two.run_until(Duration::from_secs(10), |two| two.node.asked() == Some(wired)), "the wired link verifies its lease and is asked again"); + let from = two.sent.len(); + two.resolve("three.example"); + assert_eq!(two.queries(from), [(wired, WIRED_NET.dns)]); +} + +/// Whichever link goes down last, lookups keep asking the link they asked while it was up. +#[test] +fn with_no_link_up_lookups_ask_the_lease_they_asked() { + let mut two = Two::leased(); + let (wired, wireless) = (two.wired, two.wireless); + two.link(wireless, false); + two.link(wired, false); + assert_eq!(two.node.asked(), Some(wired), "the wireless link down first, then the wired one it asked"); + let mut two = Two::leased(); + two.link(wired, false); + assert_eq!(two.node.asked(), Some(wireless), "failed over"); + two.link(wireless, false); + assert_eq!(two.node.asked(), Some(wireless), "the wired link down first: the wireless one it then asked"); +} + +/// RFC 6762 §14's laptop: a wired and a wireless link bridged onto one LAN. Each link hears +/// the other's group frames and the server's replies to both, and the machine claims its name +/// on both and refuses no server's reply. +#[test] +fn on_one_lan_by_two_links_the_name_is_claimed_on_both_and_no_reply_is_refused() { + let mut two = Two::bridged().leasing(); + let (wired, wireless) = (two.wired, two.wireless); + assert_eq!(two.node.lease(wired).map(|l| l.address), Some(LAN_WIRED.addr)); + assert_eq!(two.node.lease(wireless).map(|l| l.address), Some(LAN_WIRELESS.addr)); + two.node.answer_as(two.now, Host::new("toyos").unwrap(), counter(&mut two.draws)).unwrap(); + two.pump(); + let mut events: Vec = two.node.drain_events().collect(); + let claimed = |events: &[Event], link| events.contains(&Event::Name { link, event: toyos_mdns::Event::Claimed }); + assert!(two.run_until(Duration::from_secs(10), |two| { + events.extend(two.node.drain_events()); + claimed(&events, wired) && claimed(&events, wireless) + }), "claimed on both links: {events:?}"); + assert!(!events.iter().any(|e| matches!(e, Event::Name { event: toyos_mdns::Event::Lost, .. })), "{events:?}"); + assert!(!events.iter().any(|e| matches!(e, Event::Dhcp { .. })), "no server's reply refused: {events:?}"); + assert!(two.node.shard().ip().counters().get(toyos_net_ip::Counter::IpOwnSource) > 0, "the premise: each link heard the other's messages, which [ip] refused"); + assert!(two.node.counters().get(toyos_net_node::Counter::DhcpOtherLink) > 0, "the premise: each link heard the other's replies"); +} + +const NAME: &[u8] = b"\x05toyos\x05local\x00"; + +/// Another host's response naming `address` for this machine's name. +fn says(address: Ipv4Addr) -> Vec { + let mut m = Vec::new(); + for word in [0u16, 0x8400, 0, 1, 0, 0] { + m.extend_from_slice(&word.to_be_bytes()); + } + m.extend_from_slice(NAME); + m.extend_from_slice(&[0, 1, 0x80, 1]); + m.extend_from_slice(&120u32.to_be_bytes()); + m.extend_from_slice(&[0, 4]); + m.extend_from_slice(&address.octets()); + m +} + +/// The mDNS messages to the group that left by `link`, each by its source. +fn to_group(two: &Two, link: IfIndex) -> Vec { + two.sent + .iter() + .filter_map(|(at, out)| match out { + Out::Udp { source, destination, port: MDNS, .. } if *at == link && *destination == GROUP => Some(*source), + _ => None, + }) + .collect() +} + +#[test] +fn the_name_is_probed_for_on_each_link_from_that_links_address_and_lost_on_one_alone() { + let mut two = Two::leased(); + let (wired, wireless) = (two.wired, two.wireless); + two.node.answer_as(two.now, Host::new("toyos").unwrap(), counter(&mut two.draws)).unwrap(); + two.pump(); + let mut events = Vec::new(); + let mut conflicted = false; + let claimed = two.run_until(Duration::from_secs(10), |two| { + if !conflicted && !to_group(two, wireless).is_empty() { + // Another host on the wireless network answers for the name under its probe. + let neighbour = Ipv4Addr::new(198, 51, 100, 99); + let frame = udp_frame(GROUP_MAC, OTHER_HOST_MAC, (neighbour, MDNS), (GROUP, MDNS), &says(neighbour)); + two.deliver(wireless, &frame); + conflicted = true; + } + events.extend(two.node.drain_events()); + events.contains(&Event::Name { link: wired, event: toyos_mdns::Event::Claimed }) + }); + assert!(claimed, "the wired link's name is claimed: {events:?}"); + assert!(events.contains(&Event::Name { link: wireless, event: toyos_mdns::Event::Lost }), "lost on the wireless link: {events:?}"); + assert!(!events.contains(&Event::Name { link: wireless, event: toyos_mdns::Event::Claimed })); + assert!(to_group(&two, wired).iter().all(|&source| source == WIRED_NET.addr)); + assert!(to_group(&two, wireless).iter().all(|&source| source == WIRELESS_NET.addr)); + assert!(!to_group(&two, wired).is_empty(), "probed for on the wired link"); +} diff --git a/system/netstack/node/tests/listeners.rs b/system/netstack/node/tests/listeners.rs index 6b9b180bdcf..672fd2d69e2 100644 --- a/system/netstack/node/tests/listeners.rs +++ b/system/netstack/node/tests/listeners.rs @@ -150,6 +150,7 @@ fn draw(draws: &mut u32) -> impl FnMut() -> u32 + '_ { struct Net { node: Node, + link: toyos_net_node::IfIndex, now: Instant, draws: u32, /// The addresses a peer is at: 192.0.2.2, and each one a SYN has come from. @@ -161,8 +162,8 @@ struct Net { impl Net { /// The node holding its lease of 192.0.2.1/24 for an hour, with `common::PLACES` places. fn new() -> Self { - let Wire { node, now, .. } = Wire::leased(&terms(3_600, Some(R))); - Self { node, now, draws: 0x7e00_0000, at: BTreeSet::from([B]), heard: Vec::new() } + let Wire { node, link, now, .. } = Wire::leased(&terms(3_600, Some(R))); + Self { node, link, now, draws: 0x7e00_0000, at: BTreeSet::from([B]), heard: Vec::new() } } /// Offers the node all the credit it wants until it sends nothing more, its ARP requests @@ -170,13 +171,13 @@ impl Net { fn pump(&mut self) { for _ in 0..100_000 { let mut frames = Vec::new(); - self.node.transmit(self.now, usize::MAX, |frame| frames.push(frame.to_vec()), draw(&mut self.draws)); + self.node.transmit(self.now, usize::MAX, |_, frame| frames.push(frame.to_vec()), draw(&mut self.draws)); if frames.is_empty() { return; } for frame in &frames { if let Some(answer) = self.hears(frame) { - self.node.receive(self.now, BUDGET, batch(&[&answer]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[&answer]), draw(&mut self.draws)); } } } @@ -212,7 +213,7 @@ impl Net { } fn deliver(&mut self, frame: &[u8]) { - self.node.receive(self.now, BUDGET, batch(&[frame]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[frame]), draw(&mut self.draws)); self.pump(); } diff --git a/system/netstack/node/tests/name.rs b/system/netstack/node/tests/name.rs index 888ac14cdbd..93ddb9a2e87 100644 --- a/system/netstack/node/tests/name.rs +++ b/system/netstack/node/tests/name.rs @@ -54,7 +54,7 @@ fn named() -> Lan { /// What the node logged of its name since the last call, in order. fn said(lan: &mut Lan) -> Vec { let name = |event| match event { - toyos_net_node::Event::Name(event) => Some(event), + toyos_net_node::Event::Name { event, .. } => Some(event), _ => None, }; lan.node.drain_events().filter_map(name).collect() @@ -256,7 +256,7 @@ fn a_held_lease_is_probed_for_three_times_and_its_name_announced_only_then() { let mut lan = named(); lan.acknowledge(&terms(3_600, Some(R))); assert!(lan.datagrams().is_empty(), "an address under probe has no name yet"); - assert!(lan.run_until(Duration::from_secs(10), |lan| lan.node.lease().is_some()), "the lease is held"); + assert!(lan.run_until(Duration::from_secs(10), |lan| lan.node.lease(lan.link).is_some()), "the lease is held"); let held = lan.now; assert_eq!(said(&mut lan), [], "the name is not this machine's yet"); let asked = to_group(MAC_B, (B, 53_000), &ours(7, CLASS_IN)); @@ -301,7 +301,7 @@ fn the_first_probe_follows_the_delay_the_callers_draw_gives() { lan.fire(quiet); lan.link(false); let back = lan.now; - lan.node.link(back, true, || 1_000); + lan.node.link(back, lan.link, true, || 1_000); lan.pump(); assert_eq!(lan.datagrams().len(), 5, "nothing before the delay is over"); until(&mut lan, 6, SECOND); @@ -325,7 +325,7 @@ fn a_held_name_is_probed_for_again_when_the_link_returns_and_announced_only_then lan.link(true); let back = lan.now; - assert!(lan.node.lease().is_some(), "the lease outlived the link"); + assert!(lan.node.lease(lan.link).is_some(), "the lease outlived the link"); until(&mut lan, from + 1, SECOND); lan.deliver(&to_group(MAC_B, (B, 53_000), &ours(7, CLASS_IN))); assert_eq!(since(&lan, from), [multicast(probe())], "probed for, and answered to nobody under the probe"); @@ -612,12 +612,12 @@ fn the_name_is_answered_only_while_its_lease_is_held() { lan.deliver(&asked); assert!(lan.datagrams().is_empty(), "under probe"); - assert!(lan.run_until(Duration::from_secs(10), |lan| lan.node.lease().is_some()), "the lease is held"); + assert!(lan.run_until(Duration::from_secs(10), |lan| lan.node.lease(lan.link).is_some()), "the lease is held"); until(&mut lan, 5, Duration::from_secs(3)); lan.deliver(&asked); assert_eq!(since(&lan, 5), [unicast(53_000, legacy_response(7))], "held, and its name claimed"); - assert!(lan.run_until(Duration::from_secs(700), |lan| lan.node.lease().is_none()), "the lease runs out"); + assert!(lan.run_until(Duration::from_secs(700), |lan| lan.node.lease(lan.link).is_none()), "the lease runs out"); let from = lan.datagrams().len(); lan.deliver(&asked); assert!(since(&lan, from).is_empty(), "gone"); diff --git a/system/netstack/node/tests/resolve.rs b/system/netstack/node/tests/resolve.rs index 2ccd9389f36..7be89c32224 100644 --- a/system/netstack/node/tests/resolve.rs +++ b/system/netstack/node/tests/resolve.rs @@ -225,7 +225,7 @@ impl Net { fn leased_for(seconds: u32, servers: &[Ipv4Addr], router: Option) -> Self { let mut lan = Lan::new(); lan.lease_on(&lease_of(seconds, servers, router)); - assert_eq!(lan.node.lease().expect("held").dns, servers, "the premise: the lease names these resolvers"); + assert_eq!(lan.node.lease(lan.link).expect("held").dns, servers, "the premise: the lease names these resolvers"); // `toyos-dns` keeps whole milliseconds and conflict detection's waits are drawn finer: the // origin is the next whole one, so a wait counted from it ends on the millisecond named. let whole = u64::try_from(lan.now.since(Instant::from_millis(0)).as_millis()).unwrap() + 1; @@ -741,7 +741,7 @@ fn a_lookup_with_no_resolver_to_ask_is_refused_as_not_connected() { let mut lan = Lan::new(); lan.lease(600); lan.node.resolve(lan.now, ONE, name("www.example"), sequence([1, 2])).expect("the premise: a held lease's resolver is asked"); - assert!(lan.run_until(Duration::from_secs(700), |lan| lan.node.lease().is_none()), "the lease runs out"); + assert!(lan.run_until(Duration::from_secs(700), |lan| lan.node.lease(lan.link).is_none()), "the lease runs out"); assert_eq!(lan.node.resolve(lan.now, ONE, name("www.example"), undrawn), Err(NotStarted::NotConnected), "after the lease"); } @@ -837,14 +837,14 @@ fn a_query_with_no_port_to_leave_from_ends_its_lookup_by_name() { fn a_burst_at_a_resolver_not_yet_resolved_is_on_the_wire_once_the_resolver_answers_arp() { let mut net = Net::leased(&[ANSWERS], Some(R)); let names: Vec = (0..MAX_LOOKUPS).map(|n| format!("n{n}.example")).collect(); - let ip = |net: &Net| net.lan.node.shard().ip().neighbour(net.lan.node.shard().iface(), ANSWERS).map(toyos_net_ip::Nud::mac); + let ip = |net: &Net| net.lan.node.shard().ip().neighbour(net.lan.link, ANSWERS).map(toyos_net_ip::Nud::mac); assert_eq!(ip(&net), None, "the premise: [ip] holds no entry for the resolver"); for asked in &names { net.resolve(asked).unwrap(); } assert!(net.queried.is_empty(), "the premise: nothing has left"); assert_eq!(net.lan.opportunity(), 1, "the request for the resolver, and no query"); - let held = match net.lan.node.shard().ip().neighbour(net.lan.node.shard().iface(), ANSWERS) { + let held = match net.lan.node.shard().ip().neighbour(net.lan.link, ANSWERS) { Some(toyos_net_ip::Nud::Incomplete(asking)) => Some(asking.pending.queued()), _ => None, }; @@ -1009,7 +1009,7 @@ fn the_query_asked_in_a_dropped_querys_place_leaves_in_the_same_opportunity() { assert_eq!(net.run(first, 10 * WAIT_MS), Some(Ok(vec![ADDRESS]))); net.until(3_000); let shard = net.lan.node.shard(); - assert!(matches!(shard.ip().neighbour(shard.iface(), SILENT), Some(toyos_net_ip::Nud::Failed)), "the premise: [ip] gave the first resolver up"); + assert!(matches!(shard.ip().neighbour(net.lan.link, SILENT), Some(toyos_net_ip::Nud::Failed)), "the premise: [ip] gave the first resolver up"); let asked = net.queried.len(); let id = net.resolve("other.example").unwrap(); @@ -1052,7 +1052,7 @@ fn lease_of(seconds: u32, servers: &[Ipv4Addr], router: Option) -> Vec #[test] fn a_lookup_outlives_a_renewal_that_names_the_same_resolvers() { let (mut net, id, renewal) = renewing_under_a_lookup(); - let expiry = |net: &Net| net.lan.node.lease().and_then(|lease| lease.timers).map(|timers| timers.expiry); + let expiry = |net: &Net| net.lan.node.lease(net.lan.link).and_then(|lease| lease.timers).map(|timers| timers.expiry); let before = expiry(&net); net.lan.deliver(&from_server(MAC, A, &message_of(ACK, renewal, &naming(ANSWERS)))); assert_ne!(expiry(&net), before, "the premise: the renewal was taken, and the lease runs from it"); @@ -1068,7 +1068,7 @@ fn a_lookup_outlives_a_renewal_that_names_the_same_resolvers() { fn a_lookup_ends_when_its_leases_resolvers_change() { let (mut net, id, renewal) = renewing_under_a_lookup(); net.lan.deliver(&from_server(MAC, A, &message_of(ACK, renewal, &naming(SILENT)))); - assert_eq!(net.lan.node.lease().expect("held").dns, [SILENT], "the premise: the renewed lease names another resolver"); + assert_eq!(net.lan.node.lease(net.lan.link).expect("held").dns, [SILENT], "the premise: the renewed lease names another resolver"); assert_eq!(net.lan.node.take_resolved(), [Resolved { id, result: Err(Ended::LeaseChanged) }]); let left = net.queried[0].udp.source_port; assert!(!net.port_held(left), "port {left} of the ended lookup"); @@ -1079,7 +1079,7 @@ fn a_lookup_ends_when_its_leases_resolvers_change() { fn a_lookup_ends_when_its_lease_is_lost() { let (mut net, id, renewal) = renewing_under_a_lookup(); net.lan.deliver(&from_server(common::BROADCAST, Ipv4Addr::BROADCAST, &message(NAK, renewal, Ipv4Addr::UNSPECIFIED, &[(54, R.octets().to_vec())]))); - assert_eq!(net.lan.node.lease(), None, "the premise: the lease is gone"); + assert_eq!(net.lan.node.lease(net.lan.link), None, "the premise: the lease is gone"); assert_eq!(net.lan.node.take_resolved(), [Resolved { id, result: Err(Ended::LeaseChanged) }]); } @@ -1166,7 +1166,7 @@ fn a_link_that_goes_down_empties_what_every_view_keeps_though_the_lease_stays() net.until(100_000); let requests = net.lan.sent[up..].iter().filter(|(_, seen)| matches!(seen, Seen::Udp(udp) if udp.port == 67)).count(); assert_eq!(requests, usize::try_from(toyos_dhcp::limits::REQUEST_TRANSMISSIONS).unwrap(), "the premise: every INIT-REBOOT went unanswered"); - assert_eq!(net.lan.node.lease().map(|lease| lease.dns.clone()), Some(vec![ANSWERS]), "the premise: the lease is in use, its resolver the same"); + assert_eq!(net.lan.node.lease(net.lan.link).map(|lease| lease.dns.clone()), Some(vec![ANSWERS]), "the premise: the lease is in use, its resolver the same"); asks_again(&mut net); } @@ -1176,14 +1176,14 @@ fn a_link_that_goes_down_empties_what_every_view_keeps_though_the_lease_stays() fn a_lease_lost_and_another_naming_the_same_resolver_empty_what_every_view_keeps() { let (mut net, renewal) = renewing(); net.lan.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message(NAK, renewal, Ipv4Addr::UNSPECIFIED, &[(54, R.octets().to_vec())]))); - assert_eq!(net.lan.node.lease(), None, "the premise: the lease is gone"); + assert_eq!(net.lan.node.lease(net.lan.link), None, "the premise: the lease is gone"); let discover = last_dhcp(&net); assert_eq!(option(&discover, 53), Some(&[DISCOVER][..])); let terms = lease_of(600, &[ANSWERS], Some(R)); net.lan.deliver(&from_server(MAC, A, &message_of(OFFER, xid(&discover), &terms))); net.lan.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, xid(&discover), &terms))); - assert!(net.lan.run_until(Duration::from_secs(10), |lan| lan.node.lease().is_some()), "the new lease is held"); - assert_eq!(net.lan.node.lease().expect("held").dns, [ANSWERS], "the premise: it names the same resolver"); + assert!(net.lan.run_until(Duration::from_secs(10), |lan| lan.node.lease(lan.link).is_some()), "the new lease is held"); + assert_eq!(net.lan.node.lease(net.lan.link).expect("held").dns, [ANSWERS], "the premise: it names the same resolver"); asks_again(&mut net); } @@ -1192,7 +1192,7 @@ fn a_lease_lost_and_another_naming_the_same_resolver_empty_what_every_view_keeps #[test] fn a_renewal_with_the_same_terms_empties_what_every_view_keeps() { let (mut net, renewal) = renewing(); - let expiry = |net: &Net| net.lan.node.lease().and_then(|lease| lease.timers).map(|timers| timers.expiry); + let expiry = |net: &Net| net.lan.node.lease(net.lan.link).and_then(|lease| lease.timers).map(|timers| timers.expiry); let before = expiry(&net); net.lan.deliver(&from_server(MAC, A, &message_of(ACK, renewal, &lease_of(600, &[ANSWERS], Some(R))))); assert_ne!(expiry(&net), before, "the premise: the renewal was taken"); diff --git a/system/netstack/node/tests/slirp.rs b/system/netstack/node/tests/slirp.rs index e142c646ac3..43e94e439bf 100644 --- a/system/netstack/node/tests/slirp.rs +++ b/system/netstack/node/tests/slirp.rs @@ -109,9 +109,9 @@ fn slirps_ack_is_the_lease_its_bytes_name() { let mut wire = requesting(); let requested = wire.now; wire.deliver(&hex(SLIRP_ACK)); - assert_eq!((wire.node.lease(), wire.address(LEASED)), (None, Some(AddrState::Tentative))); - assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease().is_some()), "the lease is held"); - let lease = wire.node.lease().unwrap(); + assert_eq!((wire.node.lease(wire.link), wire.address(LEASED)), (None, Some(AddrState::Tentative))); + assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease(wire.link).is_some()), "the lease is held"); + let lease = wire.node.lease(wire.link).unwrap(); assert_eq!((lease.address, lease.prefix_len, lease.router, lease.dns.as_slice(), lease.server), (LEASED, 24, Some(SERVER), &[RESOLVER][..], SERVER)); let expiry: Option = lease.timers.map(|timers| timers.expiry); assert_eq!(expiry, Some(requested.after(Duration::from_secs(86_400)))); @@ -133,6 +133,6 @@ fn the_lease_is_read_from_the_acks_own_mask() { // With none (RFC 768), the changed mask is the lease's. ack[UDP_CHECKSUM].fill(0); wire.deliver(&ack); - assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease().is_some()), "the lease is held"); - assert_eq!(wire.node.lease().map(|lease| lease.prefix_len), Some(23)); + assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease(wire.link).is_some()), "the lease is held"); + assert_eq!(wire.node.lease(wire.link).map(|lease| lease.prefix_len), Some(23)); } diff --git a/system/netstack/node/tests/streams.rs b/system/netstack/node/tests/streams.rs index 5a6d2c1d97b..cad6c41ade1 100644 --- a/system/netstack/node/tests/streams.rs +++ b/system/netstack/node/tests/streams.rs @@ -19,7 +19,7 @@ use std::time::Duration; use common::client::Client; use common::{arp, batch, terms, Segment, Wire, A, BUDGET, MAC, MAC_B, MAC_R, R}; use etherparse::{ArpOperation, LinkSlice, NetSlice, PacketBuilder, SlicedPacket, TcpOptionElement, TransportSlice}; -use toyos::net::stream::RESET; +use toyos::net::stream::{ADDRESS_GONE, RESET}; use toyos_net_node::{ConnectRefused, ExportRefused, Node, StreamEvent, StreamId}; use toyos_net_shard::ConnectError; use toyos_net_tcp::{Counter, Endpoint, Failure}; @@ -257,6 +257,7 @@ impl Far { struct Net { node: Node, + link: toyos_net_node::IfIndex, now: Instant, draws: u32, far: Far, @@ -276,8 +277,8 @@ fn draw(draws: &mut u32) -> impl FnMut() -> u32 + '_ { impl Net { /// The node holding its lease of 192.0.2.1/24 for an hour. fn new() -> Self { - let Wire { node, now, .. } = Wire::leased(&terms(3_600, Some(R))); - Self { node, now, draws: 0x7c00_0000, far: Far::new(), parked: Vec::new(), clients: Vec::new() } + let Wire { node, link, now, .. } = Wire::leased(&terms(3_600, Some(R))); + Self { node, link, now, draws: 0x7c00_0000, far: Far::new(), parked: Vec::new(), clients: Vec::new() } } /// Leaves the present peer with the connection it has and makes `next` the peer the test @@ -321,17 +322,17 @@ impl Net { /// the opportunity is over. Returns how many frames left. fn opportunity(&mut self) -> usize { let mut frames = Vec::new(); - self.node.transmit(self.now, usize::MAX, |frame| frames.push(frame.to_vec()), draw(&mut self.draws)); + self.node.transmit(self.now, usize::MAX, |_, frame| frames.push(frame.to_vec()), draw(&mut self.draws)); for frame in &frames { for answer in self.hears(frame) { - self.node.receive(self.now, BUDGET, batch(&[&answer]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[&answer]), draw(&mut self.draws)); } } frames.len() } fn deliver(&mut self, frame: &[u8]) { - self.node.receive(self.now, BUDGET, batch(&[frame]), draw(&mut self.draws)); + self.node.receive(self.now, self.link, BUDGET, batch(&[frame]), draw(&mut self.draws)); self.pump(); } @@ -532,7 +533,7 @@ fn a_batch_of_frames_moves_an_established_stream_once_with_no_opportunity_after_ assert_eq!(client.read(64), Err(Refused::WouldBlock), "the read arms its want"); let frames = [net.far.text(b"all "), net.far.text(b"at "), net.far.text(b"once")]; let rung = client.rung(); - net.node.receive(net.now, BUDGET, batch(&[&frames[0], &frames[1], &frames[2]]), draw(&mut net.draws)); + net.node.receive(net.now, net.link, BUDGET, batch(&[&frames[0], &frames[1], &frames[2]]), draw(&mut net.draws)); assert_eq!(client.rung() - rung, 1, "one ring for the batch"); assert_eq!(inbox(&client), b"all at once"); } @@ -550,13 +551,13 @@ fn a_batch_stops_at_its_budget_and_says_a_pass_is_owed() { sink(&frame); true }; - let owed = net.node.receive(net.now, BUDGET, endless, draw(&mut net.draws)); + let owed = net.node.receive(net.now, net.link, BUDGET, endless, draw(&mut net.draws)); assert_eq!((owed, handed), (true, BUDGET), "the batch is the budget, and a pass is owed"); assert_eq!(inbox(&client), b"again", "the rest of the batch was the same segment again"); let next = net.far.text(b"once"); let short: Vec<&[u8]> = vec![&next; BUDGET - 1]; - assert!(!net.node.receive(net.now, BUDGET, batch(&short), draw(&mut net.draws)), "the ring was emptied"); + assert!(!net.node.receive(net.now, net.link, BUDGET, batch(&short), draw(&mut net.draws)), "the ring was emptied"); assert_eq!(inbox(&client), b"once"); } @@ -830,6 +831,23 @@ fn a_reset_after_the_clients_shutdown_is_the_end_the_client_reads() { assert_eq!(reads(&client), (Vec::new(), Err(Refused::Failed(RESET)))); } +// A connection cannot move to another address, so one whose address the machine loses ends +// at once, named for it, rather than retransmitting until it gives up: here another host takes +// the address after one defence (RFC 5227 §2.4 (b)). Nothing is sent from the lost address. +#[test] +fn a_stream_whose_address_is_lost_ends_as_address_gone() { + let (mut net, _, client) = established(); + let conflict = arp([0xff; 6], false, MAC_B, A, A); + net.fire(net.now.after(Duration::from_secs(100))); + net.deliver(&conflict); + net.fire(net.now.after(Duration::from_secs(5))); + let heard = net.far.segments.len(); + net.deliver(&conflict); + assert_eq!(net.node.lease(net.link), None, "the premise: the address is lost"); + assert_eq!(reads(&client), (Vec::new(), Err(Refused::Failed(ADDRESS_GONE)))); + assert_eq!(net.far.segments.len(), heard, "no reset from an address the machine gave up"); +} + // RFC 9293 §3.10.7.4 and §3.10.2: a reset after the peer's FIN flushes what was queued, and a // write after it, with room in the send ring, answers the reset and copies nothing. #[test] @@ -892,7 +910,7 @@ fn a_reset_in_the_batch_of_the_text_before_it_is_read_after_the_text() { let (mut net, _, client) = established(); let text = net.far.text(b"last"); let rst = net.far.rst(); - net.node.receive(net.now, BUDGET, batch(&[&text, &rst]), draw(&mut net.draws)); + net.node.receive(net.now, net.link, BUDGET, batch(&[&text, &rst]), draw(&mut net.draws)); assert_eq!(reads(&client), (b"last".to_vec(), Err(Refused::Failed(RESET)))); } diff --git a/system/netstack/shard/Cargo.toml b/system/netstack/shard/Cargo.toml index 72f53410b32..75a370c3960 100644 --- a/system/netstack/shard/Cargo.toml +++ b/system/netstack/shard/Cargo.toml @@ -3,7 +3,7 @@ name = "toyos-net-shard" version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" -description = "One shard of ToyOS's network stack, pure: toyos-net-ip, toyos-net-tcp and toyos-net-udp over one interface, frames in and frames pulled out with the device's credit." +description = "One shard of ToyOS's network stack, pure: toyos-net-ip, toyos-net-tcp and toyos-net-udp over every interface the caller adds, frames in by interface and frames pulled out with each link's credit." publish = false [dependencies] diff --git a/system/netstack/shard/src/lib.rs b/system/netstack/shard/src/lib.rs index a5df372ac4d..4b4bf9be3fa 100644 --- a/system/netstack/shard/src/lib.rs +++ b/system/netstack/shard/src/lib.rs @@ -1,11 +1,13 @@ -//! One shard of ToyOS's stack: `toyos-net-ip`, `toyos-net-tcp` and `toyos-net-udp` over one -//! interface. Frames come in through [`Shard::receive`] and leave only when the device pulls them -//! with [`Shard::transmit`]; the caller hands in the time and the secrets, and nothing here reads -//! a clock, draws randomness or does I/O. +//! One shard of ToyOS's stack: `toyos-net-ip`, `toyos-net-tcp` and `toyos-net-udp` over every +//! interface the caller adds ([`Shard::add_interface`]), one per link. Frames come in through +//! [`Shard::receive`] and leave only when the links pull them with [`Shard::transmit`]; the +//! caller hands in the time and the secrets, and nothing here reads a clock, draws randomness or +//! does I/O. //! -//! **Egress.** Each frame of credit goes to [ip]'s own frames first — ARP, IGMP, ICMP and the -//! datagrams resolution released — then to what [tcp] owes outside a connection, -//! and otherwise to the round: a deficit round-robin in bytes over every TCP connection and UDP +//! **Egress.** One opportunity serves every link with room, each frame handed out with the +//! interface it leaves by and spending that link's credit. A frame of credit goes to [ip]'s own +//! frames first — ARP, IGMP, ICMP and the datagrams resolution released — then to what [tcp] +//! owes outside a connection, and otherwise to the round: a deficit round-robin in bytes over every TCP connection and UDP //! sender with something to send, in the order each became eligible (RFC 8290 §3, without its new-flows list). A flow //! waiting for its next hop leaves the round, and rejoins at its tail when woken. A TCP segment is //! built only once its next hop's link address is known, and committed only once its frame is @@ -15,9 +17,11 @@ //! refused, that the table has room: a waiting flow costs one question per such change. A UDP //! datagram [ip] does not take, its next hop unresolved, stays in its sender's queue in [udp], //! spending nothing and charged nothing, while the sender's datagrams to other next hops leave: -//! it is handed to [ip] again when [ip] reports that hop resolved or the routes changed, and -//! dropped by [udp] when [ip] reports the hop failed. One [ip] refuses, for want of a route or -//! for a next hop it holds failed or has no room for, is [ip]'s to count and report. +//! it is handed to [ip] again when [ip] reports the hop on its interface resolved or the routes +//! changed, and dropped by [udp] when [ip] reports the hop on its interface failed. One [ip] refuses, for want of a route or +//! for a next hop it holds failed or has no room for, is [ip]'s to count and report. A flow +//! whose route leaves by a link with no room waits on that link as on a next hop, and asks again +//! at the next opportunity that link has room in. //! //! **Refusals.** Each crate's refusals of legacy or insecure input pass through that crate's //! `RefusalLog` here: at most one [`Event::Refused`] per rule in any 10 s, carrying how many @@ -47,7 +51,7 @@ use alloc::collections::{BTreeMap, BTreeSet, VecDeque}; use alloc::vec::Vec; use core::net::Ipv4Addr; -use toyos_net_ip::{Advice, Delivery, ErrorKind, IfIndex, Ip, Limiter, NextHop, Nud, Resolution, Sent, Source, Transport, TransportError, FRAME}; +use toyos_net_ip::{Advice, Delivery, ErrorKind, Ip, Limiter, NextHop, Nud, Resolution, Sent, Source, Transport, TransportError, FRAME}; use toyos_net_tcp::{ConnId, Endpoint, Hop, IcmpError, IcmpKind, Lent, ListenerId, Outgoing, Received, Seq, Served, Status, Tcp, Tuple}; use toyos_net_udp::{Offer, Sender, SocketId, Udp, Verdict}; use toyos_net_udp::Served as UdpServed; @@ -56,6 +60,9 @@ use toyos_net_wire::ipv4::{Ipv4Builder, Ipv4Source, MulticastAddr, TrafficClass, use toyos_net_wire::siphash::Key; use toyos_net_wire::{Instant, Port}; +pub use toyos_net_ip::{Credit, IfIndex, Rank}; +pub use toyos_net_udp::Origin; + /// Independent 128-bit keys from the system's randomness: one leaked value predicts no other. #[derive(Clone, Debug)] pub struct Secrets { @@ -67,7 +74,6 @@ pub struct Secrets { #[derive(Clone, Debug)] pub struct Config { - pub mac: IndividualMac, pub receive_buffer: u32, pub send_buffer: u32, pub secrets: Secrets, @@ -86,10 +92,11 @@ pub enum Refusal { pub enum Event { /// A line for the log, and how many refusals of its rule it stands for beyond itself. Refused { refusal: Refusal, suppressed: u64 }, - Verified(Ipv4Addr), - Conflict { addr: Ipv4Addr, mac: MacAddr }, - Lost { addr: Ipv4Addr, mac: MacAddr }, - NotVerified(Ipv4Addr), + Verified { iface: IfIndex, addr: Ipv4Addr }, + Conflict { iface: IfIndex, addr: Ipv4Addr, mac: MacAddr }, + /// Lost to another host after one defence and removed; every TCP connection from it ended. + Lost { iface: IfIndex, addr: Ipv4Addr, mac: MacAddr }, + NotVerified { iface: IfIndex, addr: Ipv4Addr }, } /// Why an active open was refused. @@ -118,15 +125,20 @@ const TCP_MTU: u16 = { u16::from_le_bytes([low, high]) }; -/// What a waiting flow waits on: its next hop's entry, or room in a full neighbour table. +/// What a waiting flow waits on: its next hop's entry, room in a full neighbour table, or room +/// on the link its route leaves by. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] enum Wait { - Hop(Ipv4Addr), + Hop(IfIndex, Ipv4Addr), Room, + Link(IfIndex), } /// A TCP segment's way out, as the hop question answered it. struct Via { + iface: IfIndex, + /// `iface`'s own link address. + source: IndividualMac, next_hop: Ipv4Addr, mac: MacAddr, } @@ -171,8 +183,8 @@ struct Log { pub struct Shard { ip: Ip, - iface: IfIndex, - mac: IndividualMac, + /// Every interface added, in the order it was. + ifaces: Vec, tcp: Tcp, udp: Udp, resets: Limiter, @@ -184,17 +196,17 @@ pub struct Shard { turn: bool, frame: Box<[u8; FRAME]>, /// The remote addresses of the TCP flows each wait holds, each until [ip] reports a change - /// for it; a flow with no route waits on the routes. + /// for it, and under a link's wait the next hops of the UDP datagrams too; a flow with no + /// route waits on the routes. waiting: BTreeMap>, /// [ip]'s routing generation when the waiting flows last asked. routes: u64, } impl Shard { - /// One interface, link down and with no address: the shell configures it. + /// No interface: the shell adds one per link. pub fn new(now: Instant, config: Config) -> Result { - let mut ip = Ip::new(now, config.secrets.ip); - let iface = ip.add_interface(now, config.mac); + let ip = Ip::new(now, config.secrets.ip); let tcp = Tcp::new(toyos_net_tcp::Config { mtu: TCP_MTU, receive_buffer: config.receive_buffer, @@ -204,8 +216,7 @@ impl Shard { Ok(Self { routes: ip.generation(), ip, - iface, - mac: config.mac, + ifaces: Vec::new(), tcp, udp: Udp::new(), resets: Limiter::new(config.secrets.resets), @@ -222,9 +233,11 @@ impl Shard { &self.ip } - /// The interface the shard's [`Ip`] names. - pub fn iface(&self) -> IfIndex { - self.iface + /// A link's interface, down and with no address: the shell configures it. + pub fn add_interface(&mut self, now: Instant, mac: IndividualMac, rank: Rank) -> IfIndex { + let iface = self.ip.add_interface(now, mac, rank); + self.ifaces.push(iface); + iface } pub fn tcp_counters(&self) -> &toyos_net_tcp::Counters { @@ -242,9 +255,9 @@ impl Shard { // ---- the device ---- - /// A frame the device received. - pub fn receive(&mut self, now: Instant, frame: &[u8]) { - match self.ip.receive(now, self.iface, frame) { + /// A frame `iface`'s link received. + pub fn receive(&mut self, now: Instant, iface: IfIndex, frame: &[u8]) { + match self.ip.receive(now, iface, frame) { Some(Delivery::Tcp(arrival, segment)) => { let resets = &mut self.resets; let (from, to) = (arrival.packet.source(), arrival.packet.destination()); @@ -264,20 +277,37 @@ impl Shard { self.settle(now); } - /// A transmit opportunity with room for `credit` frames, each handed to `sink` as it is built. - /// Returns how many left. - pub fn transmit(&mut self, now: Instant, credit: usize, mut sink: impl FnMut(&[u8])) -> usize { + /// A transmit opportunity with `credit`'s room on each link, each frame handed to `sink` as + /// it is built, with the interface it leaves by. Returns how many left. + pub fn transmit(&mut self, now: Instant, mut credit: impl Credit, mut sink: impl FnMut(IfIndex, &[u8])) -> usize { + let credit = &mut credit; + let room: Vec = self.ifaces.iter().copied().filter(|&iface| credit.room(iface)).collect(); + for iface in room { + for hop in self.waiting.remove(&Wait::Link(iface)).into_iter().flatten() { + self.tcp.wake(hop); + self.udp.wake(iface, hop); + } + } + self.settle(now); let mut spent = 0usize; - while spent < credit { - let sent = self.ip.transmit(now, 1, |_, frame| sink(frame)) > 0 - || self.tcp_frame(now, &mut sink, |tcp, hop, out| tcp.transmit_owed(now, 1, hop, out)).1.is_some() - || self.round_frame(now, &mut sink) - // A flow that waits has queued the request it waits on. - || self.ip.transmit(now, 1, |_, frame| sink(frame)) > 0; - if !sent { + while self.ifaces.iter().any(|&iface| credit.room(iface)) { + let own = self.ip.transmit(now, &mut *credit, &mut sink); + spent = spent.saturating_add(own); + let sent = own > 0 + || self.tcp_frame(now, credit, &mut sink, |tcp, hop, out| tcp.transmit_owed(now, 1, hop, out)).1.is_some() + || self.round_frame(now, credit, &mut sink); + if sent && own == 0 { + spent = spent.saturating_add(1); + } + if sent { + continue; + } + // A flow that waits has queued the request it waits on. + let requests = self.ip.transmit(now, &mut *credit, &mut sink); + spent = spent.saturating_add(requests); + if requests == 0 { break; } - spent = spent.saturating_add(1); } self.settle(now); spent @@ -288,7 +318,7 @@ impl Shard { /// deficit is above 0, each frame's length charged as it leaves, and then goes to the tail /// keeping its deficit. A flow with nothing left leaves the round, its deficit with it: a UDP /// sender as its last datagram is handed out, a TCP connection once served with nothing due. - fn round_frame(&mut self, now: Instant, sink: &mut impl FnMut(&[u8])) -> bool { + fn round_frame(&mut self, now: Instant, credit: &mut impl Credit, sink: &mut impl FnMut(IfIndex, &[u8])) -> bool { loop { let Some(head) = self.round.front_mut() else { return false }; if !core::mem::replace(&mut self.turn, true) { @@ -296,12 +326,12 @@ impl Shard { } let flow = head.flow; let Outcome { frame, leaves } = match flow { - Flow::Tcp(id) => match self.tcp_frame(now, sink, |tcp, hop, out| tcp.serve(now, id, hop, out)) { + Flow::Tcp(id) => match self.tcp_frame(now, credit, sink, |tcp, hop, out| tcp.serve(now, id, hop, out)) { (Served::Sent, Some(len)) => Outcome { frame: Some(len), leaves: false }, (Served::Done, None) => Outcome { frame: None, leaves: true }, (Served::Sent, None) | (Served::Done, Some(_)) => unreachable!("[tcp] answers Sent exactly when the sink framed a segment"), }, - Flow::Udp(sender) => self.udp_frame(now, sender, sink), + Flow::Udp(sender) => self.udp_frame(now, sender, credit, sink), }; if let (Some(len), Some(head)) = (frame, self.round.front_mut()) { head.deficit = head.deficit.saturating_sub(len); @@ -325,18 +355,20 @@ impl Shard { fn tcp_frame( &mut self, now: Instant, - sink: &mut impl FnMut(&[u8]), + credit: &mut impl Credit, + sink: &mut impl FnMut(IfIndex, &[u8]), send: impl FnOnce(&mut Tcp, &mut dyn FnMut(&Tuple) -> Hop, &mut dyn FnMut(&Outgoing<'_>, Via)) -> R, ) -> (R, Option) { - let Self { ip, iface, mac, tcp, frame, waiting, .. } = self; - let iface = *iface; + let Self { ip, tcp, frame, waiting, .. } = self; let mut built = None; - let done = send(tcp, &mut |tuple| hop(ip, now, iface, tuple, waiting), &mut |out, via| { - let Some(bytes) = tcp_datagram(out, *mac, via.mac, frame) else { unreachable!("TCP's MTU, [tcp]'s bounds and [ip]'s addresses fit every segment in a frame") }; - sink(bytes); - built = Some((via.next_hop, out.source, charge(bytes))); + let room = &*credit; + let done = send(tcp, &mut |tuple| hop(ip, now, room, tuple, waiting), &mut |out, via| { + let Some(bytes) = tcp_datagram(out, via.source, via.mac, frame) else { unreachable!("TCP's MTU, [tcp]'s bounds and [ip]'s addresses fit every segment in a frame") }; + sink(via.iface, bytes); + built = Some((via.iface, via.next_hop, out.source, charge(bytes))); }); - let Some((next_hop, source, len)) = built else { return (done, None) }; + let Some((iface, next_hop, source, len)) = built else { return (done, None) }; + credit.spend(iface); // The send the neighbour machine counts: STALE moves to DELAY (RFC 4861 §7.3.3). ip.resolve(now, iface, next_hop, source); (done, Some(len)) @@ -344,22 +376,30 @@ impl Shard { /// One datagram of `sender`'s, framed by [ip]: the oldest whose next hop's link address is /// known, or that needs none. One [ip] refuses as it leaves makes no frame. - fn udp_frame(&mut self, now: Instant, sender: Sender, sink: &mut impl FnMut(&[u8])) -> Outcome { - let Self { ip, udp, frame, .. } = self; + fn udp_frame(&mut self, now: Instant, sender: Sender, credit: &mut impl Credit, sink: &mut impl FnMut(IfIndex, &[u8])) -> Outcome { + let Self { ip, udp, frame, waiting, .. } = self; let mut sent = None; - let served = udp.serve(sender, |out| match ip.send_udp(now, out, frame) { - Ok(Sent::Frame(len)) => { + let room = &*credit; + let served = udp.serve(sender, |out| match ip.send_udp(now, room, out, frame) { + Ok(Sent::Frame { len, iface }) => { sent = frame.get(..len).map(|bytes| { - sink(bytes); - charge(bytes) + sink(iface, bytes); + (iface, charge(bytes)) }); Offer::Taken } - Ok(Sent::Pending(next_hop)) => Offer::Waits(next_hop), + Ok(Sent::Pending { iface, next_hop }) => Offer::Waits { iface, hop: next_hop }, + Ok(Sent::NoRoom { iface, hop }) => { + waiting.entry(Wait::Link(iface)).or_default().insert(hop); + Offer::Waits { iface, hop } + } // Counted by [ip], which tells the datagram's flow. Err(_) => Offer::Taken, }); - Outcome { frame: sent, leaves: served != UdpServed::More } + if let Some((iface, _)) = sent { + credit.spend(iface); + } + Outcome { frame: sent.map(|(_, len)| len), leaves: served != UdpServed::More } } pub fn next_deadline(&self) -> Option { @@ -385,8 +425,8 @@ impl Shard { events.push(Event::Refused { refusal: Refusal::Tcp(r), suppressed }); } } - toyos_net_tcp::Event::Reachable(peer) => ip.advise(now, peer, Advice::Confirmed), - toyos_net_tcp::Event::Reverify(peer) => ip.advise(now, peer, Advice::Reverify), + toyos_net_tcp::Event::Reachable { local, remote } => ip.advise(now, remote, Source::Bound(local), Advice::Confirmed), + toyos_net_tcp::Event::Reverify { local, remote } => ip.advise(now, remote, Source::Bound(local), Advice::Reverify), } } for event in ip.drain_events() { @@ -397,21 +437,24 @@ impl Shard { } } toyos_net_ip::Event::Unreachable(flow) => udp.unreachable(&flow), - toyos_net_ip::Event::Resolved { next_hop, .. } => { - wake(tcp, waiting, Wait::Hop(next_hop)); - udp.wake(next_hop); + toyos_net_ip::Event::Resolved { iface, next_hop } => { + wake(tcp, waiting, Wait::Hop(iface, next_hop)); + udp.wake(iface, next_hop); } - toyos_net_ip::Event::Failed { next_hop, .. } => { - wake(tcp, waiting, Wait::Hop(next_hop)); - udp.fail(next_hop); + toyos_net_ip::Event::Failed { iface, next_hop } => { + wake(tcp, waiting, Wait::Hop(iface, next_hop)); + udp.fail(iface, next_hop); } // No datagram waits on a failed next hop: the failure dropped them. - toyos_net_ip::Event::Cleared { next_hop, .. } => wake(tcp, waiting, Wait::Hop(next_hop)), + toyos_net_ip::Event::Cleared { iface, next_hop } => wake(tcp, waiting, Wait::Hop(iface, next_hop)), toyos_net_ip::Event::Room { .. } => wake(tcp, waiting, Wait::Room), - toyos_net_ip::Event::Verified { addr, .. } => events.push(Event::Verified(addr)), - toyos_net_ip::Event::Conflict { addr, mac, .. } => events.push(Event::Conflict { addr, mac }), - toyos_net_ip::Event::Lost { addr, mac, .. } => events.push(Event::Lost { addr, mac }), - toyos_net_ip::Event::NotVerified { addr, .. } => events.push(Event::NotVerified(addr)), + toyos_net_ip::Event::Verified { iface, addr } => events.push(Event::Verified { iface, addr }), + toyos_net_ip::Event::Conflict { iface, addr, mac } => events.push(Event::Conflict { iface, addr, mac }), + toyos_net_ip::Event::Lost { iface, addr, mac } => { + tcp.lose_address(addr); + events.push(Event::Lost { iface, addr, mac }); + } + toyos_net_ip::Event::NotVerified { iface, addr } => events.push(Event::NotVerified { iface, addr }), } } for r in udp.drain_refusals() { @@ -440,44 +483,47 @@ impl Shard { // ---- configuration, the shell's ---- - pub fn link_up(&mut self, now: Instant) -> Result<(), toyos_net_ip::Counter> { - let up = self.ip.link_up(now, self.iface); + pub fn link_up(&mut self, now: Instant, iface: IfIndex) -> Result<(), toyos_net_ip::Counter> { + let up = self.ip.link_up(now, iface); self.settle(now); up } - pub fn link_down(&mut self, now: Instant) -> Result<(), toyos_net_ip::Counter> { - let down = self.ip.link_down(now, self.iface); + pub fn link_down(&mut self, now: Instant, iface: IfIndex) -> Result<(), toyos_net_ip::Counter> { + let down = self.ip.link_down(now, iface); self.settle(now); down } /// Added tentative: conflict detection runs before it is used ([`Event::Verified`]). - pub fn add_address(&mut self, now: Instant, addr: Ipv4Addr, prefix_len: u8) -> Result<(), toyos_net_ip::Counter> { - let added = self.ip.add_address(now, self.iface, addr, prefix_len); + pub fn add_address(&mut self, now: Instant, iface: IfIndex, addr: Ipv4Addr, prefix_len: u8) -> Result<(), toyos_net_ip::Counter> { + let added = self.ip.add_address(now, iface, addr, prefix_len); self.settle(now); added } - pub fn set_gateways(&mut self, now: Instant, gateways: &[Ipv4Addr]) -> Result<(), toyos_net_ip::Counter> { - let set = self.ip.set_gateways(now, self.iface, gateways); + pub fn set_gateways(&mut self, now: Instant, iface: IfIndex, gateways: &[Ipv4Addr]) -> Result<(), toyos_net_ip::Counter> { + let set = self.ip.set_gateways(now, iface, gateways); self.settle(now); set } /// Takes one of the interface's addresses away; [ip] withdraws the gateways no usable prefix - /// holds any more. - pub fn remove_address(&mut self, now: Instant, addr: Ipv4Addr) -> Result<(), toyos_net_ip::Counter> { - let removed = self.ip.remove_address(now, self.iface, addr); + /// holds any more, and every TCP connection from it ends (`Failure::AddressGone`). + pub fn remove_address(&mut self, now: Instant, iface: IfIndex, addr: Ipv4Addr) -> Result<(), toyos_net_ip::Counter> { + let removed = self.ip.remove_address(now, iface, addr); + if removed.is_ok() { + self.tcp.lose_address(addr); + } self.settle(now); removed } - /// Joins `group` on the interface: frames to its link address are taken from here on, and - /// [ip] reports the membership whenever the link can carry the report. Nothing to settle: - /// a join makes no flow eligible and no event. - pub fn join(&mut self, now: Instant, group: MulticastAddr) -> Result<(), toyos_net_ip::Counter> { - self.ip.join(now, self.iface, group) + /// Joins `group` on `iface`: frames to its link address are taken from here on, and [ip] + /// reports the membership whenever the link can carry the report. Nothing to settle: a join + /// makes no flow eligible and no event. + pub fn join(&mut self, now: Instant, iface: IfIndex, group: MulticastAddr) -> Result<(), toyos_net_ip::Counter> { + self.ip.join(now, iface, group) } // ---- TCP ---- @@ -680,10 +726,17 @@ impl Shard { Ok(id) } - /// The acquisition socket's send, naming its source: 0.0.0.0 to the limited broadcast, or an - /// assigned address. - pub fn send_from(&mut self, now: Instant, id: SocketId, source: Ipv4Addr, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), toyos_net_udp::Error> { - let queued = self.udp.send_from(&mut self.ip, id, source, destination, port, payload); + /// [`Udp::send_via`](toyos_net_udp::Udp::send_via): a datagram that leaves by `iface` alone. + pub fn send_via(&mut self, now: Instant, id: SocketId, iface: IfIndex, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), toyos_net_udp::Error> { + let queued = self.udp.send_via(&mut self.ip, id, iface, destination, port, payload); + self.settle(now); + queued + } + + /// The acquisition socket's send, naming its source, 0.0.0.0 to the limited broadcast or an + /// assigned address, and the interface whose address it acquires. + pub fn send_from(&mut self, now: Instant, id: SocketId, origin: Origin, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), toyos_net_udp::Error> { + let queued = self.udp.send_from(&mut self.ip, id, origin, destination, port, payload); self.settle(now); queued } @@ -693,19 +746,25 @@ impl Shard { /// move: only a next hop with no entry is resolved, which queues its request. /// No route is a local destination unreachable (RFC 1122 §3.3.1.1). A flow told to wait is /// recorded under what it waits on; one with no route waits on the routes. -fn hop(ip: &mut Ip, now: Instant, iface: IfIndex, tuple: &Tuple, waiting: &mut BTreeMap>) -> Hop { +fn hop(ip: &mut Ip, now: Instant, credit: &impl Credit, tuple: &Tuple, waiting: &mut BTreeMap>) -> Hop { let remote = tuple.remote.addr; - let Ok(route) = ip.route(remote, Source::Bound(tuple.local.addr), Some(iface)) else { return Hop::Unreachable }; + let Ok(route) = ip.route(remote, Source::Bound(tuple.local.addr), None) else { return Hop::Unreachable }; let NextHop::Neighbour(next_hop) = route.next_hop else { return Hop::Unreachable }; + let iface = route.iface; + let Some(source) = ip.mac(iface) else { unreachable!("a route names an interface [ip] holds") }; + if !credit.room(iface) { + waiting.entry(Wait::Link(iface)).or_default().insert(remote); + return Hop::Pending; + } let (answer, wait) = match ip.neighbour(iface, next_hop) { Some(entry) => match entry.mac() { - Some(mac) => (Hop::Ready(Via { next_hop, mac }), None), - None if matches!(entry, Nud::Failed) => (Hop::Unreachable, Some(Wait::Hop(next_hop))), - None => (Hop::Pending, Some(Wait::Hop(next_hop))), + Some(mac) => (Hop::Ready(Via { iface, source, next_hop, mac }), None), + None if matches!(entry, Nud::Failed) => (Hop::Unreachable, Some(Wait::Hop(iface, next_hop))), + None => (Hop::Pending, Some(Wait::Hop(iface, next_hop))), }, None => match ip.resolve(now, iface, next_hop, route.source) { - Resolution::Resolved(mac) => (Hop::Ready(Via { next_hop, mac }), None), - Resolution::Pending => (Hop::Pending, Some(Wait::Hop(next_hop))), + Resolution::Resolved(mac) => (Hop::Ready(Via { iface, source, next_hop, mac }), None), + Resolution::Pending => (Hop::Pending, Some(Wait::Hop(iface, next_hop))), Resolution::Failed => (Hop::Unreachable, Some(Wait::Room)), }, }; @@ -776,11 +835,12 @@ mod tests { }; let mac = IndividualMac::new(MacAddr([2, 0, 0, 0, 0, 0x0a])).unwrap(); let mut now = Instant::from_millis(3_600_000); - let mut shard = Shard::new(now, Config { mac, receive_buffer: 65_535, send_buffer: 65_535, secrets }).unwrap(); - shard.link_up(now).unwrap(); - shard.add_address(now, A, 24).unwrap(); - while !shard.drain_events().any(|e| e == Event::Verified(A)) { - shard.transmit(now, usize::MAX, |_| {}); + let mut shard = Shard::new(now, Config { receive_buffer: 65_535, send_buffer: 65_535, secrets }).unwrap(); + let iface = shard.add_interface(now, mac, Rank(0)); + shard.link_up(now, iface).unwrap(); + shard.add_address(now, iface, A, 24).unwrap(); + while !shard.drain_events().any(|e| matches!(e, Event::Verified { addr: A, .. })) { + shard.transmit(now, usize::MAX, |_, _| {}); now = shard.next_deadline().expect("conflict detection is still running"); shard.fire(now); } @@ -809,13 +869,13 @@ mod tests { const C: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 3); const D: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 4); let (mut shard, now) = verified(); - shard.transmit(now, usize::MAX, |_| {}); + shard.transmit(now, usize::MAX, |_, _| {}); let port = |n| Port::new(n).unwrap(); let (to_c, to_d) = (shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5000)), || 0).unwrap(), shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5001)), || 0).unwrap()); shard.send_to(now, to_c, C, 9, b"c").unwrap(); shard.send_to(now, to_d, D, 9, b"d").unwrap(); assert_eq!(shard.round.len(), 2); - assert_eq!(shard.transmit(now, usize::MAX, |_| {}), 2, "a request for each"); + assert_eq!(shard.transmit(now, usize::MAX, |_, _| {}), 2, "a request for each"); assert!(shard.round.is_empty(), "what waits is not served"); let mut reply = [[2, 0, 0, 0, 0, 0x0a], [2, 0, 0, 0, 0, 0x0c]].concat(); @@ -823,7 +883,7 @@ mod tests { reply.extend_from_slice(&C.octets()); reply.extend_from_slice(&[2, 0, 0, 0, 0, 0x0a]); reply.extend_from_slice(&A.octets()); - shard.receive(now, &reply); + shard.receive(now, shard.ifaces[0], &reply); assert_eq!(shard.round.iter().map(|m| m.flow).collect::>(), [Flow::Udp(Sender::Socket(to_c))]); } diff --git a/system/netstack/shard/tcp/src/conn.rs b/system/netstack/shard/tcp/src/conn.rs index cd76d9b2705..f73b25bc400 100644 --- a/system/netstack/shard/tcp/src/conn.rs +++ b/system/netstack/shard/tcp/src/conn.rs @@ -779,7 +779,7 @@ impl Sync { self.acked_since = Some(now); self.stalled = 0; self.orphan_progress(now); - ctx.log.event(Event::Reachable(ctx.tuple.remote.addr)); + ctx.log.event(Event::Reachable { local: ctx.tuple.local.addr, remote: ctx.tuple.remote.addr }); let smss = self.smss(); match self.recovery { Recovery::Fast { .. } if ack.sub(1).at_or_after(self.recover) => { @@ -1225,7 +1225,7 @@ impl Sync { self.stalled = self.stalled.saturating_add(1); if self.stalled == 3 { self.delivery_problem = true; - ctx.log.event(Event::Reverify(ctx.tuple.remote.addr)); + ctx.log.event(Event::Reverify { local: ctx.tuple.local.addr, remote: ctx.tuple.remote.addr }); } } diff --git a/system/netstack/shard/tcp/src/counters.rs b/system/netstack/shard/tcp/src/counters.rs index 5211c3faf39..19865bb0566 100644 --- a/system/netstack/shard/tcp/src/counters.rs +++ b/system/netstack/shard/tcp/src/counters.rs @@ -116,18 +116,18 @@ impl Log { /// Past [`limits::EVENTS`] undrained, an event is refused and counted: a peer never grows /// the list. A `Reachable` is not repeated while it is the newest advice pending for its - /// address: one peer's ACKs never crowd out another's refusals, and [ip] still reads each - /// address's advice in order. + /// pair of addresses: one peer's ACKs never crowd out another's refusals, and [ip] still + /// reads each pair's advice in order. pub fn event(&mut self, event: Event) { match event { - Event::Reachable(addr) if self.newest_advice(addr) == Some(&event) => {} + Event::Reachable { local, remote } if self.newest_advice(local, remote) == Some(&event) => {} _ if self.events.len() >= limits::EVENTS => self.count(Counter::EventOverflow), _ => self.events.push(event), } } - fn newest_advice(&self, addr: Ipv4Addr) -> Option<&Event> { - self.events.iter().rev().find(|e| matches!(e, Event::Reachable(a) | Event::Reverify(a) if *a == addr)) + fn newest_advice(&self, local: Ipv4Addr, remote: Ipv4Addr) -> Option<&Event> { + self.events.iter().rev().find(|e| matches!(e, Event::Reachable { local: l, remote: r } | Event::Reverify { local: l, remote: r } if *l == local && *r == remote)) } pub fn drain(&mut self) -> alloc::vec::Drain<'_, Event> { diff --git a/system/netstack/shard/tcp/src/lib.rs b/system/netstack/shard/tcp/src/lib.rs index 05a27c8cb9e..8662de061d1 100644 --- a/system/netstack/shard/tcp/src/lib.rs +++ b/system/netstack/shard/tcp/src/lib.rs @@ -212,6 +212,8 @@ pub enum Failure { Unreachable(SoftError), /// ICMP destination unreachable, administratively prohibited, during the handshake. Prohibited, + /// The host gave up the connection's local address ([`Tcp::lose_address`]). + AddressGone, } /// An ICMP error that did not end the connection; reported as the reason if it later gives up. @@ -314,10 +316,11 @@ impl Hop { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Event { Refused(Refusal), - /// An ACK advanced SND.UNA: the next hop to this address answers (RFC 1122 §3.3.1.4). - Reachable(Ipv4Addr), - /// Three expiries without progress: re-verify the next hop to this address. - Reverify(Ipv4Addr), + /// An ACK advanced SND.UNA: the next hop to `remote` from `local` answers (RFC 1122 + /// §3.3.1.4). The local address names the link: one remote may be reached by several. + Reachable { local: Ipv4Addr, remote: Ipv4Addr }, + /// Three expiries without progress: re-verify the next hop to `remote` from `local`. + Reverify { local: Ipv4Addr, remote: Ipv4Addr }, } // Each compile_fail block sits beside one that compiles, so a typo cannot pass it. diff --git a/system/netstack/shard/tcp/src/open.rs b/system/netstack/shard/tcp/src/open.rs index 7322f850390..644a10e2cc4 100644 --- a/system/netstack/shard/tcp/src/open.rs +++ b/system/netstack/shard/tcp/src/open.rs @@ -145,7 +145,7 @@ impl Retransmit { self.owed = true; self.stalled = self.stalled.saturating_add(1); if self.stalled == 3 { - ctx.log.event(crate::Event::Reverify(ctx.tuple.remote.addr)); + ctx.log.event(crate::Event::Reverify { local: ctx.tuple.local.addr, remote: ctx.tuple.remote.addr }); } false } diff --git a/system/netstack/shard/tcp/src/stack.rs b/system/netstack/shard/tcp/src/stack.rs index fe9c0b0f5e6..aef8287d958 100644 --- a/system/netstack/shard/tcp/src/stack.rs +++ b/system/netstack/shard/tcp/src/stack.rs @@ -1112,6 +1112,22 @@ impl Tcp { Ok(()) } + /// The host no longer holds `local`: every connection from it ends with + /// [`Failure::AddressGone`], with no reset, since nothing can be sent from it. A connection + /// cannot move to another address, so waiting would only end it later by its timeout. + pub fn lose_address(&mut self, local: Ipv4Addr) { + let lost: Vec = self + .conns + .iter() + .enumerate() + .filter_map(|(index, slot)| slot.value.as_ref().filter(|c| c.tuple.local.addr == local && !matches!(c.state, Tcb::Ended(_))).map(|_| index)) + .filter_map(|index| u32::try_from(index).ok()) + .collect(); + for index in lost { + self.end(index, Some(Failure::AddressGone), None); + } + } + /// RFC 9293 §3.10.5. pub fn abort(&mut self, now: Instant, id: ConnId) -> Result<(), Error> { let conn = self.conn(id)?; diff --git a/system/netstack/shard/tcp/tests/close.rs b/system/netstack/shard/tcp/tests/close.rs index 75e96652fef..8da10d95866 100644 --- a/system/netstack/shard/tcp/tests/close.rs +++ b/system/netstack/shard/tcp/tests/close.rs @@ -459,3 +459,18 @@ fn s_cl_032_rst_in_last_ack_and_closing() { nothing(&h.input(10, seg(5002).rst())); assert!(!held(&mut h)); } + +/// A connection whose local address the host gave up ends at once, named for +/// it, and sends nothing: no reset can leave from an address it no longer +/// holds. One from another address is untouched. +#[test] +fn a_connection_from_a_lost_address_ends_and_sends_nothing() { + let mut h = fixture_e(); + let other = h.tcp.connect(h.now(), std::net::Ipv4Addr::new(198, 51, 100, 1), Some(port(49153)), ep(B, 80)).unwrap(); + h.transmit(); + h.tcp.lose_address(A); + nothing(&h.transmit()); + assert_eq!(h.call(10, |tcp, now, id| tcp.send(now, id, b"x")).0, Err(Error::Failed(Failure::AddressGone))); + let status = h.tcp.status(other).unwrap(); + assert_eq!(status.state, State::SynSent, "the other address's connection is left"); +} diff --git a/system/netstack/shard/tcp/tests/modern.rs b/system/netstack/shard/tcp/tests/modern.rs index 1ac1dc56680..6e879679b8e 100644 --- a/system/netstack/shard/tcp/tests/modern.rs +++ b/system/netstack/shard/tcp/tests/modern.rs @@ -88,7 +88,7 @@ fn s_mod_005_a_bulk_peer_leaves_room_for_refusals() { h.arrive(seg(7000).syn().rst().from(Ipv4Addr::new(192, 0, 2, 3), 1234).to(A, 81)); h.transmit(); assert_eq!(h.info().snd_una.get(), 1001 + limits::EVENTS as u32 + 1); - assert_eq!(h.events.iter().filter(|e| **e == Event::Reachable(B)).count(), 1); + assert_eq!(h.events.iter().filter(|e| **e == Event::Reachable { local: A, remote: B }).count(), 1); assert_eq!(h.refusals(Counter::SynRst).len(), 1); assert_eq!(h.count(Counter::EventOverflow), 0); } diff --git a/system/netstack/shard/tcp/tests/timers.rs b/system/netstack/shard/tcp/tests/timers.rs index 133a5f07153..76688ff0a88 100644 --- a/system/netstack/shard/tcp/tests/timers.rs +++ b/system/netstack/shard/tcp/tests/timers.rs @@ -92,10 +92,10 @@ fn s_gu_001_three_expiries_raise_advice() { h.send(0, 100); h.at(1399); assert!(!h.status().delivery_problem); - assert!(!h.events.contains(&Event::Reverify(B))); + assert!(!h.events.contains(&Event::Reverify { local: A, remote: B })); h.at(1400); assert!(h.status().delivery_problem); - assert_eq!(h.events.iter().filter(|e| **e == Event::Reverify(B)).count(), 1); + assert_eq!(h.events.iter().filter(|e| **e == Event::Reverify { local: A, remote: B }).count(), 1); } #[test] @@ -163,7 +163,7 @@ fn s_gu_006_progress_confirms_the_next_hop() { h.input(10, seg(5001).ack(2461)); h.input(11, seg(5001).ack(2461)); h.input(12, seg(5001).ack(3921)); - assert_eq!(h.events.iter().filter(|e| **e == Event::Reachable(B)).count(), 2); + assert_eq!(h.events.iter().filter(|e| **e == Event::Reachable { local: A, remote: B }).count(), 2); } /// [ip] reads the advice in order: an advance after a pending re-verify confirms the next hop again. @@ -182,7 +182,7 @@ fn s_gu_006_a_confirmation_after_a_reverify_is_kept() { } h.arrive(seg(5001).ack(3921)); let events: Vec = h.tcp.drain_events().collect(); - assert_eq!(events, [Event::Reachable(B), Event::Reverify(B), Event::Reachable(B)]); + assert_eq!(events, [Event::Reachable { local: A, remote: B }, Event::Reverify { local: A, remote: B }, Event::Reachable { local: A, remote: B }]); } /// [ip] reads each address's own advice: B's repeated confirmation never crowds out C's, and a @@ -204,5 +204,5 @@ fn s_gu_006_b_advice_is_kept_per_address() { h.arrive(seg(5001).ack(1101).from(C, 80).to(A, 49153)); h.arrive(seg(5001).ack(1201)); let events: Vec = h.tcp.drain_events().collect(); - assert_eq!(events, [Event::Reachable(B), Event::Reachable(C)]); + assert_eq!(events, [Event::Reachable { local: A, remote: B }, Event::Reachable { local: A, remote: C }]); } diff --git a/system/netstack/shard/testnet/Cargo.toml b/system/netstack/shard/testnet/Cargo.toml index 6b21b363e4c..86f05c229c3 100644 --- a/system/netstack/shard/testnet/Cargo.toml +++ b/system/netstack/shard/testnet/Cargo.toml @@ -7,6 +7,7 @@ description = "A fake Ethernet for toyos-net-shard's host tests: N shards on one publish = false [dependencies] +toyos-net-ip = { path = "../../ip" } toyos-net-shard = { path = ".." } toyos-net-tcp = { path = "../tcp" } toyos-net-wire = { path = "../../wire" } diff --git a/system/netstack/shard/testnet/src/lib.rs b/system/netstack/shard/testnet/src/lib.rs index e91e3d6a056..f56d6bfc349 100644 --- a/system/netstack/shard/testnet/src/lib.rs +++ b/system/netstack/shard/testnet/src/lib.rs @@ -19,7 +19,7 @@ use std::collections::BTreeMap; use std::net::Ipv4Addr; use std::time::Duration; -use toyos_net_shard::{Config, Event, Secrets, Shard}; +use toyos_net_shard::{Config, Event, IfIndex, Rank, Secrets, Shard}; use toyos_net_tcp::{ConnId, Endpoint, Error, Failure, ListenerId, Received, Tuple}; use toyos_net_wire::ethernet::{Frame, IndividualMac, MacAddr, MacClass}; use toyos_net_wire::{Instant, Port}; @@ -82,6 +82,8 @@ impl Link { pub struct Node { pub shard: Shard, + /// Its one interface, on the segment. + pub iface: IfIndex, pub mac: MacAddr, pub credit: Credit, /// What the shard reported, oldest first. @@ -89,6 +91,33 @@ pub struct Node { budget: (u64, usize), } +impl Node { + pub fn receive(&mut self, now: Instant, frame: &[u8]) { + self.shard.receive(now, self.iface, frame); + } + + pub fn link_up(&mut self, now: Instant) -> Result<(), toyos_net_ip::Counter> { + self.shard.link_up(now, self.iface) + } + + pub fn link_down(&mut self, now: Instant) -> Result<(), toyos_net_ip::Counter> { + self.shard.link_down(now, self.iface) + } + + pub fn add_address(&mut self, now: Instant, addr: Ipv4Addr, prefix_len: u8) -> Result<(), toyos_net_ip::Counter> { + self.shard.add_address(now, self.iface, addr, prefix_len) + } + + pub fn set_gateways(&mut self, now: Instant, gateways: &[Ipv4Addr]) -> Result<(), toyos_net_ip::Counter> { + self.shard.set_gateways(now, self.iface, gateways) + } + + /// Whether the shard has reported `addr` verified. + pub fn verified(&self, addr: Ipv4Addr) -> bool { + self.events.iter().any(|e| matches!(e, Event::Verified { addr: a, .. } if *a == addr)) + } +} + /// A frame as a node's device put it on the wire. #[derive(Clone, Debug)] pub struct Carried { @@ -245,15 +274,16 @@ impl Net { port_table: [0; 16], }, }; - let config = Config { mac: IndividualMac::new(mac).expect("an individual MAC"), receive_buffer: 65_535, send_buffer: 65_535, secrets }; + let config = Config { receive_buffer: 65_535, send_buffer: 65_535, secrets }; let mut shard = Shard::new(self.now, config).expect("a valid configuration"); - shard.link_up(self.now).expect("its own interface"); - shard.add_address(self.now, addr, prefix_len).expect("a valid address"); + let iface = shard.add_interface(self.now, IndividualMac::new(mac).expect("an individual MAC"), Rank(0)); + shard.link_up(self.now, iface).expect("its own interface"); + shard.add_address(self.now, iface, addr, prefix_len).expect("a valid address"); for other in 0..index { self.links.insert((index, other), Link::default()); self.links.insert((other, index), Link::default()); } - self.nodes.push(Node { shard, mac, credit: Credit::Unlimited, events: Vec::new(), budget: (u64::MAX, 0) }); + self.nodes.push(Node { shard, iface, mac, credit: Credit::Unlimited, events: Vec::new(), budget: (u64::MAX, 0) }); index } @@ -319,7 +349,8 @@ impl Net { let due = self.due(); let mut busy = !due.is_empty(); for flight in due { - self.nodes[flight.to].shard.receive(self.now, &flight.frame); + let node = &mut self.nodes[flight.to]; + node.shard.receive(self.now, node.iface, &flight.frame); } for node in &mut self.nodes { node.shard.fire(self.now); @@ -363,7 +394,8 @@ impl Net { } }; let mut frames = Vec::new(); - let spent = node.shard.transmit(now, credit, |frame| frames.push(frame.to_vec())); + let mut credit = credit; + let spent = node.shard.transmit(now, &mut credit, |_, frame| frames.push(frame.to_vec())); if let Credit::PerMs(_) = node.credit { node.budget.1 -= spent; } diff --git a/system/netstack/shard/tests/acquisition.rs b/system/netstack/shard/tests/acquisition.rs index 2b0258dd788..2e53bc46101 100644 --- a/system/netstack/shard/tests/acquisition.rs +++ b/system/netstack/shard/tests/acquisition.rs @@ -7,7 +7,7 @@ mod common; use std::net::Ipv4Addr; use common::*; -use toyos_net_shard::{Config, Event, Secrets, Shard}; +use toyos_net_shard::{Config, Event, IfIndex, Origin, Rank, Secrets, Shard}; use toyos_net_udp::{Counter, Error}; use toyos_net_wire::arp::Arp; use toyos_net_wire::ethernet::{EtherType, Frame, IndividualMac, MacAddr}; @@ -16,7 +16,7 @@ use toyos_net_wire::udp::UdpDatagram; use toyos_net_wire::Instant; /// A's shard an hour into the clock, its link up and no address on it. -fn unaddressed() -> (Shard, Instant) { +fn unaddressed() -> (Shard, IfIndex, Instant) { let secrets = Secrets { ip: [1; 16], resets: [2; 16], @@ -24,21 +24,22 @@ fn unaddressed() -> (Shard, Instant) { }; let now = Instant::from_millis(3_600_000); let mac = IndividualMac::new(MAC_A).unwrap(); - let mut shard = Shard::new(now, Config { mac, receive_buffer: 65_535, send_buffer: 65_535, secrets }).unwrap(); - shard.link_up(now).unwrap(); - (shard, now) + let mut shard = Shard::new(now, Config { receive_buffer: 65_535, send_buffer: 65_535, secrets }).unwrap(); + let iface = shard.add_interface(now, mac, Rank(0)); + shard.link_up(now, iface).unwrap(); + (shard, iface, now) } fn sent(shard: &mut Shard, now: Instant) -> Vec> { let mut frames = Vec::new(); - shard.transmit(now, usize::MAX, |frame| frames.push(frame.to_vec())); + shard.transmit(now, usize::MAX, |_, frame| frames.push(frame.to_vec())); frames } /// A's address verified, and the time it was. -fn verified(shard: &mut Shard, mut now: Instant) -> Instant { - shard.add_address(now, A, 24).unwrap(); - while !shard.drain_events().any(|e| e == Event::Verified(A)) { +fn verified(shard: &mut Shard, iface: IfIndex, mut now: Instant) -> Instant { + shard.add_address(now, iface, A, 24).unwrap(); + while !shard.drain_events().any(|e| matches!(e, Event::Verified { addr: A, .. })) { sent(shard, now); now = shard.next_deadline().expect("conflict detection is still running"); shard.fire(now); @@ -80,9 +81,9 @@ fn who_has(target: Ipv4Addr) -> Vec { #[test] fn the_acquisition_socket_sends_from_no_address_to_the_limited_broadcast() { - let (mut shard, now) = unaddressed(); + let (mut shard, iface, now) = unaddressed(); let socket = shard.acquisition().unwrap(); - shard.send_from(now, socket, Ipv4Addr::UNSPECIFIED, Ipv4Addr::BROADCAST, 67, b"discover").unwrap(); + shard.send_from(now, socket, Origin { iface, source: Ipv4Addr::UNSPECIFIED }, Ipv4Addr::BROADCAST, 67, b"discover").unwrap(); let frames = sent(&mut shard, now); let [frame] = &frames[..] else { panic!("one frame, not {}", frames.len()) }; let frame = Frame::parse(frame).unwrap(); @@ -95,9 +96,9 @@ fn the_acquisition_socket_sends_from_no_address_to_the_limited_broadcast() { #[test] fn the_acquisition_socket_hears_a_datagram_to_an_address_not_yet_held() { - let (mut shard, now) = unaddressed(); + let (mut shard, iface, now) = unaddressed(); let socket = shard.acquisition().unwrap(); - shard.receive(now, &from_router(MAC_A, A, b"offer")); + shard.receive(now, iface, &from_router(MAC_A, A, b"offer")); let mut buf = [0u8; 16]; let received = shard.recv_from(socket, &mut buf).unwrap().expect("the acquisition exception admits it"); assert_eq!((&buf[..received.len], received.source, received.source_port), (&b"offer"[..], R, Some(port(67)))); @@ -105,30 +106,30 @@ fn the_acquisition_socket_hears_a_datagram_to_an_address_not_yet_held() { #[test] fn only_the_acquisition_socket_names_its_source() { - let (mut shard, now) = unaddressed(); - let now = verified(&mut shard, now); + let (mut shard, iface, now) = unaddressed(); + let now = verified(&mut shard, iface, now); sent(&mut shard, now); let plain = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5000)), || 0).unwrap(); let acquisition = shard.acquisition().unwrap(); // A source the acquisition socket may name: an assigned address, to one host. - assert_eq!(shard.send_from(now, plain, A, R, 67, b"request"), Err(Error::Refused(Counter::SourceNotPermitted))); - assert_eq!(shard.send_from(now, acquisition, A, R, 67, b"request"), Ok(())); + assert_eq!(shard.send_from(now, plain, Origin { iface, source: A }, R, 67, b"request"), Err(Error::Refused(Counter::SourceNotPermitted))); + assert_eq!(shard.send_from(now, acquisition, Origin { iface, source: A }, R, 67, b"request"), Ok(())); } #[test] fn a_second_acquisition_socket_is_refused_its_port() { - let (mut shard, _) = unaddressed(); + let (mut shard, _, _) = unaddressed(); shard.acquisition().unwrap(); assert_eq!(shard.acquisition(), Err(Error::Refused(Counter::PortInUse))); } #[test] fn a_removed_address_takes_its_gateway_and_answers_nothing() { - let (mut shard, now) = unaddressed(); - let now = verified(&mut shard, now); - shard.set_gateways(now, &[R]).unwrap(); + let (mut shard, iface, now) = unaddressed(); + let now = verified(&mut shard, iface, now); + shard.set_gateways(now, iface, &[R]).unwrap(); sent(&mut shard, now); - shard.receive(now, &who_has(A)); + shard.receive(now, iface, &who_has(A)); let answered = sent(&mut shard, now); let replies = |frames: &[Vec]| { let arp = |f: &Vec| Frame::parse(f).ok().filter(|f| f.ether_type() == EtherType::Arp).and_then(|f| Arp::parse(f.body()).ok()); @@ -136,10 +137,10 @@ fn a_removed_address_takes_its_gateway_and_answers_nothing() { }; assert_eq!(replies(&answered), 1, "held, the address answers"); - shard.remove_address(now, A).unwrap(); - assert_eq!(shard.ip().address(shard.iface(), A), None); - assert_eq!(shard.ip().gateways(shard.iface()), Some(&[] as &[Ipv4Addr])); - shard.receive(now, &who_has(A)); + shard.remove_address(now, iface, A).unwrap(); + assert_eq!(shard.ip().address(iface, A), None); + assert_eq!(shard.ip().gateways(iface), Some(&[] as &[Ipv4Addr])); + shard.receive(now, iface, &who_has(A)); assert_eq!(replies(&sent(&mut shard, now)), 0, "removed, it answers nothing"); - assert_eq!(shard.remove_address(now, A), Err(toyos_net_ip::Counter::AddrUnknown)); + assert_eq!(shard.remove_address(now, iface, A), Err(toyos_net_ip::Counter::AddrUnknown)); } diff --git a/system/netstack/shard/tests/common/mod.rs b/system/netstack/shard/tests/common/mod.rs index f54d5823daf..357ad019f9c 100644 --- a/system/netstack/shard/tests/common/mod.rs +++ b/system/netstack/shard/tests/common/mod.rs @@ -8,7 +8,7 @@ use std::net::Ipv4Addr; use std::time::Duration; -use toyos_net_shard::Event; + use toyos_net_tcp::{ConnId, Endpoint, State}; use toyos_net_testnet::Net; use toyos_net_wire::arp::{Arp, Operation}; @@ -39,10 +39,10 @@ pub fn segment() -> (Net, usize, usize) { let mut net = Net::new(Instant::from_millis(3_600_000)); let a = net.add_node(MAC_A, A, 24); let b = net.add_node(MAC_B, B, 24); - let verified = |net: &mut Net, n: usize, addr| net.nodes[n].events.contains(&Event::Verified(addr)); + let verified = |net: &mut Net, n: usize, addr| net.nodes[n].verified(addr); assert!(net.run_until(Duration::from_secs(1), |net| verified(net, a, A) && verified(net, b, B)), "both addresses verified"); net.advance(toyos_net_ip::limits::acd::ANNOUNCE_INTERVAL * 2); - let none = |n: usize, addr| net.nodes[n].shard.ip().neighbour(net.nodes[n].shard.iface(), addr).is_none(); + let none = |n: usize, addr| net.nodes[n].shard.ip().neighbour(net.nodes[n].iface, addr).is_none(); assert!(none(a, B) && none(b, A), "no neighbour entries yet"); (net, a, b) } diff --git a/system/netstack/shard/tests/drr.rs b/system/netstack/shard/tests/drr.rs index 3a7f8fc60a7..3d271736187 100644 --- a/system/netstack/shard/tests/drr.rs +++ b/system/netstack/shard/tests/drr.rs @@ -10,7 +10,7 @@ use std::net::Ipv4Addr; use std::time::Duration; use common::*; -use toyos_net_shard::Event; + use toyos_net_tcp::{ConnId, State}; use toyos_net_testnet::{Credit, Net}; use toyos_net_wire::ethernet::MacAddr; @@ -39,7 +39,7 @@ fn fixture() -> Fixture { let mut net = Net::new(Instant::from_millis(3_600_000)); let a = net.add_node(MAC_A, A, 24); let peers = [(net.add_node(MAC_B, B, 24), B), (net.add_node(MAC_C, C, 24), C), (net.add_node(MAC_D, D, 24), D)]; - let verified = |net: &mut Net| [(a, A)].into_iter().chain(peers).all(|(n, addr)| net.nodes[n].events.contains(&Event::Verified(addr))); + let verified = |net: &mut Net| [(a, A)].into_iter().chain(peers).all(|(n, addr)| net.nodes[n].verified(addr)); assert!(net.run_until(Duration::from_secs(1), verified), "every address verified"); net.advance(toyos_net_ip::limits::acd::ANNOUNCE_INTERVAL * 2); for (n, addr) in peers { @@ -57,7 +57,7 @@ fn fixture() -> Fixture { fn opportunity(f: &mut Fixture, credit: usize) -> Vec<(String, usize)> { let now = f.net.now(); let mut out = Vec::new(); - f.net.nodes[f.a].shard.transmit(now, credit, |frame| out.push((name(frame), frame.len()))); + f.net.nodes[f.a].shard.transmit(now, credit, |_, frame| out.push((name(frame), frame.len()))); out } @@ -164,8 +164,8 @@ fn s_shard_drr_003_ip_and_owed_resets_go_first_and_the_round_resumes() { let (mut f, first) = bulk(); assert_eq!(names(&first), [B_BULK, C_BULK, D_BULK, B_BULK]); let now = f.net.now(); - f.net.nodes[f.a].shard.receive(now, &echo_request()); - f.net.nodes[f.a].shard.receive(now, &arp_request_from_d()); + f.net.nodes[f.a].receive(now, &echo_request()); + f.net.nodes[f.a].receive(now, &arp_request_from_d()); f.net.nodes[f.a].shard.abort(now, f.conns[4]).unwrap(); let out = opportunity(&mut f, 6); assert_eq!( @@ -187,7 +187,7 @@ fn a_connection_that_takes_a_freed_slot_takes_one_turn() { let third = f.net.nodes[a].shard.connect(now, Some(port(49158)), ep(C, 81)).unwrap(); let syn = opportunity_frames(&mut f, 1); assert_eq!(syn.len(), 1); - f.net.nodes[c].shard.receive(now, &syn[0]); + f.net.nodes[c].receive(now, &syn[0]); let established = |net: &mut Net| net.nodes[a].shard.status(third).unwrap().state == State::Established; assert!(f.net.run_until(Duration::from_secs(1), established), "the SYN-ACK arrives"); let now = f.net.now(); @@ -200,7 +200,7 @@ fn a_connection_that_takes_a_freed_slot_takes_one_turn() { fn opportunity_frames(f: &mut Fixture, credit: usize) -> Vec> { let now = f.net.now(); let mut out = Vec::new(); - f.net.nodes[f.a].shard.transmit(now, credit, |frame| out.push(frame.to_vec())); + f.net.nodes[f.a].shard.transmit(now, credit, |_, frame| out.push(frame.to_vec())); out } diff --git a/system/netstack/shard/tests/egress.rs b/system/netstack/shard/tests/egress.rs index 7b07e22cccf..27c6cea50be 100644 --- a/system/netstack/shard/tests/egress.rs +++ b/system/netstack/shard/tests/egress.rs @@ -25,7 +25,7 @@ const MAC_D: MacAddr = MacAddr([0x02, 0, 0, 0, 0, 0x0d]); fn opportunity(net: &mut Net, node: usize, credit: usize) -> Vec { let now = net.now(); let mut out = Vec::new(); - net.nodes[node].shard.transmit(now, credit, |frame| out.push(name(frame))); + net.nodes[node].shard.transmit(now, credit, |_, frame| out.push(name(frame))); out } @@ -93,7 +93,7 @@ fn s_pl_013_shard_a_connect_with_no_route_fails_at_once() { let (mut net, a, _) = segment(); let now = net.now(); let id = net.nodes[a].shard.connect(now, Some(port(49152)), ep(B, 80)).unwrap(); - net.nodes[a].shard.link_down(now).unwrap(); + net.nodes[a].link_down(now).unwrap(); net.advance(Duration::from_secs(5)); assert_eq!(net.nodes[a].shard.status(id).unwrap().failure, Some(HOST_UNREACHABLE)); let counters = net.nodes[a].shard.tcp_counters(); @@ -109,9 +109,9 @@ fn routes_refused(net: &Net, node: usize) -> u64 { /// A forgets every neighbour: a link down and up again. fn forget(net: &mut Net, node: usize) { let now = net.now(); - net.nodes[node].shard.link_down(now).unwrap(); - net.nodes[node].shard.link_up(now).unwrap(); - let iface = net.nodes[node].shard.iface(); + net.nodes[node].link_down(now).unwrap(); + net.nodes[node].link_up(now).unwrap(); + let iface = net.nodes[node].iface; assert!(net.nodes[node].shard.ip().neighbour(iface, B).is_none()); } @@ -178,7 +178,7 @@ fn s_pl_014_shard_an_established_flow_with_no_route_records_host_unreachable() { let (mut net, a, b) = segment(); let (to_b, from_a) = established(&mut net, a, b); let now = net.now(); - net.nodes[a].shard.link_down(now).unwrap(); + net.nodes[a].link_down(now).unwrap(); net.nodes[a].shard.send(now, to_b, &[0x55; 100]).unwrap(); net.advance(Duration::from_secs(10)); let status = net.nodes[a].shard.status(to_b).unwrap(); @@ -187,7 +187,7 @@ fn s_pl_014_shard_an_established_flow_with_no_route_records_host_unreachable() { // A route again: the flow asks, and its segment leaves; B's acknowledgment clears the error. let now = net.now(); - net.nodes[a].shard.link_up(now).unwrap(); + net.nodes[a].link_up(now).unwrap(); net.advance(Duration::from_secs(1)); assert_eq!(net.nodes[a].shard.status(to_b).unwrap().soft_error, None); let now = net.now(); @@ -200,7 +200,7 @@ fn s_pl_014_shard_an_established_flow_with_no_route_records_host_unreachable() { fn b_failed_d_ready() -> (Net, usize, toyos_net_tcp::ConnId, toyos_net_tcp::ConnId) { let (mut net, a, b) = segment(); let d = net.add_node(MAC_D, D, 24); - assert!(net.run_until(Duration::from_secs(1), |net| net.nodes[d].events.contains(&toyos_net_shard::Event::Verified(D)))); + assert!(net.run_until(Duration::from_secs(1), |net| net.nodes[d].verified(D))); net.advance(toyos_net_ip::limits::acd::ANNOUNCE_INTERVAL * 2); let (to_b, _) = established(&mut net, a, b); let (to_d, _) = connected(&mut net, a, 49153, d, D); @@ -211,7 +211,7 @@ fn b_failed_d_ready() -> (Net, usize, toyos_net_tcp::ConnId, toyos_net_tcp::Conn net.nodes[a].shard.send_to(now, socket, B, 9, b"x").unwrap(); net.nodes[a].shard.send_to(now, socket, D, 9, b"x").unwrap(); net.advance(Duration::from_secs(4)); - let iface = net.nodes[a].shard.iface(); + let iface = net.nodes[a].iface; assert!(matches!(net.nodes[a].shard.ip().neighbour(iface, B), Some(Nud::Failed))); assert!(net.nodes[a].shard.ip().neighbour(iface, D).is_some_and(|n| n.mac().is_some())); (net, a, to_b, to_d) @@ -237,7 +237,7 @@ fn s_pl_015_shard_a_reset_with_no_route_is_dropped() { let (mut net, a, b) = segment(); let (to_b, _) = established(&mut net, a, b); let now = net.now(); - net.nodes[a].shard.link_down(now).unwrap(); + net.nodes[a].link_down(now).unwrap(); let start = net.wire().len(); net.nodes[a].shard.abort(now, to_b).unwrap(); net.advance(Duration::from_secs(5)); @@ -252,7 +252,7 @@ fn s_hs_034_shard_a_child_whose_peer_never_answers_gives_up() { let start = net.wire().len(); let now = net.now(); net.link(b, a).dark = Some((now, now.after(Duration::from_secs(3_600)))); - net.nodes[a].shard.receive(now, &from_b(&segment_bytes((B, 40_000), (A, 80), 5_000, 0x02))); + net.nodes[a].receive(now, &from_b(&segment_bytes((B, 40_000), (A, 80), 5_000, 0x02))); let given_up = |net: &Net| net.nodes[a].shard.tcp_counters().get(Counter::SynAckGiveUp); net.advance(Duration::from_millis(59_999)); assert_eq!(given_up(&net), 0); @@ -327,19 +327,19 @@ fn stale_peer() -> (Net, usize, toyos_net_tcp::ConnId) { let (to_b, _) = established(&mut net, a, b); forget(&mut net, a); let now = net.now(); - net.nodes[b].shard.link_down(now).unwrap(); - net.nodes[b].shard.link_up(now).unwrap(); + net.nodes[b].link_down(now).unwrap(); + net.nodes[b].link_up(now).unwrap(); net.nodes[a].shard.bind(A, Some(port(9)), || 0).unwrap(); let socket = net.nodes[b].shard.bind(B, None, || 0).unwrap(); net.nodes[b].shard.send_to(now, socket, A, 9, b"x").unwrap(); net.advance(Duration::from_millis(1)); - let iface = net.nodes[a].shard.iface(); + let iface = net.nodes[a].iface; assert!(matches!(net.nodes[a].shard.ip().neighbour(iface, B), Some(Nud::Stale(_)))); (net, a, to_b) } fn state_of_b(net: &Net, a: usize) -> Option<&Nud> { - net.nodes[a].shard.ip().neighbour(net.nodes[a].shard.iface(), B) + net.nodes[a].shard.ip().neighbour(net.nodes[a].iface, B) } #[test] @@ -367,10 +367,10 @@ fn s_ip_nud_009_shard_an_acknowledgment_ends_delay() { fn s_ip_nud_020_shard_a_stalled_connect_reverifies_the_gateway() { let (mut net, a, _) = segment(); let r = net.add_node(MAC_R, R, 24); - assert!(net.run_until(Duration::from_secs(1), |net| net.nodes[r].events.contains(&toyos_net_shard::Event::Verified(R)))); + assert!(net.run_until(Duration::from_secs(1), |net| net.nodes[r].verified(R))); net.advance(toyos_net_ip::limits::acd::ANNOUNCE_INTERVAL * 2); let now = net.now(); - net.nodes[a].shard.set_gateways(now, &[R]).unwrap(); + net.nodes[a].set_gateways(now, &[R]).unwrap(); let start = net.wire().len(); net.nodes[a].shard.connect(now, Some(port(49152)), ep(std::net::Ipv4Addr::new(198, 51, 100, 7), 80)).unwrap(); assert!(net.run_until(Duration::from_secs(1), |net| sent(net, a, start).iter().any(|f| f.contains("SYN")))); @@ -380,7 +380,7 @@ fn s_ip_nud_020_shard_a_stalled_connect_reverifies_the_gateway() { assert!(!sent(&net, a, start).contains(&"ARP poll 192.0.2.254".to_string()), "{}", dump(&net, start)); net.advance(Duration::from_millis(1)); assert!(sent(&net, a, start).contains(&"ARP poll 192.0.2.254".to_string()), "{}", dump(&net, start)); - let iface = net.nodes[a].shard.iface(); + let iface = net.nodes[a].iface; assert!(net.nodes[a].shard.ip().neighbour(iface, std::net::Ipv4Addr::new(198, 51, 100, 7)).is_none()); } @@ -389,7 +389,7 @@ fn s_ip_nud_022_shard_a_flow_refused_by_a_full_table_asks_again_once_an_entry_ma let mut net = Net::new(Instant::from_millis(3_600_000)); let a = net.add_node(MAC_A, A, 22); let b = net.add_node(MAC_B, B, 22); - let verified = |net: &mut Net, n: usize, addr| net.nodes[n].events.contains(&toyos_net_shard::Event::Verified(addr)); + let verified = |net: &mut Net, n: usize, addr| net.nodes[n].verified(addr); assert!(net.run_until(Duration::from_secs(1), |net| verified(net, a, A) && verified(net, b, B))); net.advance(toyos_net_ip::limits::acd::ANNOUNCE_INTERVAL * 2); let (to_b, from_a) = established(&mut net, a, b); @@ -426,14 +426,14 @@ fn s_ip_nud_024_shard_a_segments_request_prefers_its_source() { let (mut net, a, _) = segment(); let (second, peer) = (Ipv4Addr::new(198, 51, 100, 1), Ipv4Addr::new(198, 51, 100, 9)); let now = net.now(); - net.nodes[a].shard.add_address(now, second, 24).unwrap(); - assert!(net.run_until(Duration::from_secs(1), |net| net.nodes[a].events.contains(&toyos_net_shard::Event::Verified(second)))); + net.nodes[a].add_address(now, second, 24).unwrap(); + assert!(net.run_until(Duration::from_secs(1), |net| net.nodes[a].verified(second))); net.nodes[a].shard.listen(A, Some(port(80)), || 0).unwrap(); let start = net.wire().len(); let mut syn = from_b(&segment_bytes((peer, 40_000), (A, 80), 5_000, 0x02)); syn[6..12].copy_from_slice(&[0x02, 0, 0, 0, 0, 0x99]); let now = net.now(); - net.nodes[a].shard.receive(now, &syn); + net.nodes[a].receive(now, &syn); net.advance(Duration::from_millis(1)); let request = net.wire()[start..].iter().find(|c| c.from == a && name(&c.frame) == format!("ARP request {peer}")).expect("a request"); let frame = toyos_net_wire::ethernet::Frame::parse(&request.frame).unwrap(); diff --git a/system/netstack/shard/tests/icmp.rs b/system/netstack/shard/tests/icmp.rs index ec3dc3fe50d..2f16afb1d76 100644 --- a/system/netstack/shard/tests/icmp.rs +++ b/system/netstack/shard/tests/icmp.rs @@ -33,7 +33,7 @@ fn outstanding() -> (Net, usize, ConnId, Vec) { fn inject(net: &mut Net, node: usize, frame: &[u8]) { let now = net.now(); - net.nodes[node].shard.receive(now, frame); + net.nodes[node].receive(now, frame); } #[test] diff --git a/system/netstack/shard/tests/links.rs b/system/netstack/shard/tests/links.rs new file mode 100644 index 00000000000..6a3ba19e1c5 --- /dev/null +++ b/system/netstack/shard/tests/links.rs @@ -0,0 +1,334 @@ +//! One shard over two links: a new flow leaves by the link that ranks first and has a route, +//! fails over and returns with it, waits for its own link's room without holding back the +//! other's, and ends when its address is lost. + +use std::net::Ipv4Addr; + +use toyos_net_shard::{Config, Credit, Event, IfIndex, Rank, Secrets, Shard}; +use toyos_net_tcp::{Endpoint, Error, Failure}; +use toyos_net_wire::arp::{Arp, Operation}; +use toyos_net_wire::ethernet::{EtherType, Frame, FrameBuilder, IndividualMac, MacAddr}; +use toyos_net_wire::ipv4::Ipv4Packet; +use toyos_net_wire::{Instant, Port}; + +const WIRED_MAC: MacAddr = MacAddr([2, 0, 0, 0, 0, 0x01]); +const WIRELESS_MAC: MacAddr = MacAddr([2, 0, 0, 0, 0, 0x02]); +const WIRED: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 10); +const WIRED_ROUTER: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 1); +const WIRELESS: Ipv4Addr = Ipv4Addr::new(198, 51, 100, 10); +const WIRELESS_ROUTER: Ipv4Addr = Ipv4Addr::new(198, 51, 100, 1); +const ROUTER_MAC: MacAddr = MacAddr([2, 0, 0, 0, 0, 0xfe]); +/// Somewhere only a router reaches. +const FAR: Ipv4Addr = Ipv4Addr::new(203, 0, 113, 7); + +struct Machine { + shard: Shard, + wired: IfIndex, + wireless: IfIndex, + now: Instant, +} + +fn secrets() -> Secrets { + Secrets { + ip: [1; 16], + resets: [2; 16], + tcp: toyos_net_tcp::Secrets { isn: [3; 16], timestamp: [4; 16], port_offset: [5; 16], port_index: [6; 16], port_table: [0; 16] }, + } +} + +/// An ARP reply from a router at `ip` to `to`. +fn router_reply(ip: Ipv4Addr, to: (MacAddr, Ipv4Addr)) -> Vec { + let arp = Arp { operation: Operation::Reply, sender_mac: ROUTER_MAC, sender_ip: ip, target_mac: to.0, target_ip: to.1 }; + let mut frame = vec![0; 64]; + let len = FrameBuilder { destination: to.0, source: IndividualMac::new(ROUTER_MAC).unwrap() }.emit(&arp, &mut frame).unwrap().len(); + frame.truncate(len); + frame +} + +/// The wireless link added first and the wired one second, both up, each address verified +/// and its router known. +fn machine() -> Machine { + let mut now = Instant::from_millis(3_600_000); + let mut shard = Shard::new(now, Config { receive_buffer: 65_535, send_buffer: 65_535, secrets: secrets() }).unwrap(); + let wireless = shard.add_interface(now, IndividualMac::new(WIRELESS_MAC).unwrap(), Rank(1)); + let wired = shard.add_interface(now, IndividualMac::new(WIRED_MAC).unwrap(), Rank(0)); + for (iface, addr) in [(wireless, WIRELESS), (wired, WIRED)] { + shard.link_up(now, iface).unwrap(); + shard.add_address(now, iface, addr, 24).unwrap(); + } + let mut verified = 0; + while verified < 2 { + shard.transmit(now, usize::MAX, |_, _| {}); + verified += shard.drain_events().filter(|e| matches!(e, Event::Verified { .. })).count(); + if verified < 2 { + now = shard.next_deadline().expect("conflict detection is still running"); + shard.fire(now); + } + } + shard.set_gateways(now, wireless, &[WIRELESS_ROUTER]).unwrap(); + shard.set_gateways(now, wired, &[WIRED_ROUTER]).unwrap(); + Machine { shard, wired, wireless, now } +} + +fn far(port: u16) -> Endpoint { + Endpoint { addr: FAR, port: Port::new(port).unwrap() } +} + +/// The IPv4 frames that left with `credit`, each by its interface and source, a router +/// answering every ARP request for it as it leaves, until nothing more leaves. +fn left(m: &mut Machine, mut credit: impl Credit) -> Vec<(IfIndex, Ipv4Addr)> { + let mut out = Vec::new(); + loop { + let mut asked = Vec::new(); + let sent = m.shard.transmit(m.now, &mut credit, |iface, bytes| { + let frame = Frame::parse(bytes).unwrap(); + match frame.ether_type() { + EtherType::Ipv4 => out.push((iface, Ipv4Packet::parse(frame.body()).unwrap().source())), + EtherType::Arp => { + let arp = Arp::parse(frame.body()).unwrap(); + if arp.operation == Operation::Request && [WIRED_ROUTER, WIRELESS_ROUTER].contains(&arp.target_ip) { + asked.push((iface, router_reply(arp.target_ip, (arp.sender_mac, arp.sender_ip)))); + } + } + EtherType::Other(_) => {} + } + }); + for (iface, reply) in &asked { + m.shard.receive(m.now, *iface, reply); + } + if sent == 0 && asked.is_empty() { + return out; + } + } +} + +#[test] +fn a_new_flow_leaves_by_the_wired_link_though_the_wireless_came_first() { + let mut m = machine(); + m.shard.connect(m.now, None, far(80)).unwrap(); + let (wired, out) = (m.wired, left(&mut m, usize::MAX)); + assert_eq!(out, [(wired, WIRED)], "its SYN"); +} + +#[test] +fn a_new_flow_fails_over_to_the_wireless_link_and_the_next_returns_to_the_wired_one() { + let mut m = machine(); + m.shard.link_down(m.now, m.wired).unwrap(); + m.shard.connect(m.now, None, far(80)).unwrap(); + let (wireless, wired) = (m.wireless, m.wired); + assert_eq!(left(&mut m, usize::MAX), [(wireless, WIRELESS)], "failed over"); + m.shard.link_up(m.now, wired).unwrap(); + let back = left(&mut m, usize::MAX); + assert!(back.is_empty(), "nothing moves when the wired link returns: {back:?}"); + m.shard.connect(m.now, None, far(81)).unwrap(); + assert_eq!(left(&mut m, usize::MAX), [(wired, WIRED)], "the next flow is the wired link's again"); +} + +/// Room on one interface alone. +struct Only(IfIndex); + +impl Credit for Only { + fn room(&self, iface: IfIndex) -> bool { + iface == self.0 + } + fn spend(&mut self, _: IfIndex) {} +} + +#[test] +fn a_flow_waits_for_its_own_links_room_and_holds_back_none_of_the_others() { + let mut m = machine(); + let (wired, wireless) = (m.wired, m.wireless); + m.shard.connect(m.now, None, far(80)).unwrap(); + m.shard.link_down(m.now, wired).unwrap(); + m.shard.connect(m.now, None, far(81)).unwrap(); + m.shard.link_up(m.now, wired).unwrap(); + // The wired flow's SYN was due first; with room on the wireless link alone, the wireless + // flow's leaves and the wired one's waits. + assert_eq!(left(&mut m, Only(wireless)), [(wireless, WIRELESS)]); + assert_eq!(left(&mut m, Only(wired)), [(wired, WIRED)], "the wired flow, at the wired link's next room"); +} + +#[test] +fn a_flow_from_an_address_taken_away_ends_and_one_from_the_other_link_does_not() { + let mut m = machine(); + let wired_flow = m.shard.connect(m.now, None, far(80)).unwrap(); + m.shard.link_down(m.now, m.wired).unwrap(); + let wireless_flow = m.shard.connect(m.now, None, far(81)).unwrap(); + m.shard.link_up(m.now, m.wired).unwrap(); + assert_eq!(left(&mut m, usize::MAX).len(), 2, "both SYNs"); + m.shard.remove_address(m.now, m.wired, WIRED).unwrap(); + assert_eq!(m.shard.send(m.now, wired_flow, b"x"), Err(Error::Failed(Failure::AddressGone))); + assert!(m.shard.status(wireless_flow).is_ok_and(|s| s.state == toyos_net_tcp::State::SynSent)); +} + +/// A link going down takes no address, so it ends no flow as its address's loss does. +#[test] +fn a_link_that_only_goes_down_keeps_its_address() { + let mut m = machine(); + m.shard.link_down(m.now, m.wired).unwrap(); + assert!(m.shard.ip().address(m.wired, WIRED).is_some()); +} + +// ---- one LAN on two links: one gateway address on each ---- + +const LAN_GATEWAY: Ipv4Addr = Ipv4Addr::new(192, 168, 1, 1); +const LAN_WIRED: Ipv4Addr = Ipv4Addr::new(192, 168, 1, 10); +const LAN_WIRELESS: Ipv4Addr = Ipv4Addr::new(192, 168, 1, 20); + +/// Both links on one /24, each with the same gateway address, and a datagram socket bound to +/// each link's address. +fn one_lan() -> (Machine, toyos_net_udp::SocketId, toyos_net_udp::SocketId) { + let mut now = Instant::from_millis(3_600_000); + let mut shard = Shard::new(now, Config { receive_buffer: 65_535, send_buffer: 65_535, secrets: secrets() }).unwrap(); + let wired = shard.add_interface(now, IndividualMac::new(WIRED_MAC).unwrap(), Rank(0)); + let wireless = shard.add_interface(now, IndividualMac::new(WIRELESS_MAC).unwrap(), Rank(1)); + for (iface, addr) in [(wired, LAN_WIRED), (wireless, LAN_WIRELESS)] { + shard.link_up(now, iface).unwrap(); + shard.add_address(now, iface, addr, 24).unwrap(); + } + let mut verified = 0; + while verified < 2 { + shard.transmit(now, usize::MAX, |_, _| {}); + verified += shard.drain_events().filter(|e| matches!(e, Event::Verified { .. })).count(); + if verified < 2 { + now = shard.next_deadline().expect("conflict detection is still running"); + shard.fire(now); + } + } + for iface in [wired, wireless] { + shard.set_gateways(now, iface, &[LAN_GATEWAY]).unwrap(); + } + let from_wired = shard.bind(LAN_WIRED, Port::new(5000), || 0).unwrap(); + let from_wireless = shard.bind(LAN_WIRELESS, Port::new(5001), || 0).unwrap(); + (Machine { shard, wired, wireless, now }, from_wired, from_wireless) +} + +/// The UDP datagrams that left in one opportunity, by interface and source; ARP answered by +/// nobody. +fn datagrams(m: &mut Machine) -> Vec<(IfIndex, Ipv4Addr)> { + let mut out = Vec::new(); + m.shard.transmit(m.now, usize::MAX, |iface, bytes| { + let frame = Frame::parse(bytes).unwrap(); + if matches!(frame.ether_type(), EtherType::Ipv4) { + out.push((iface, Ipv4Packet::parse(frame.body()).unwrap().source())); + } + }); + out +} + +#[test] +fn a_gateway_that_fails_on_one_link_drops_no_datagram_waiting_on_the_other() { + let (mut m, from_wired, from_wireless) = one_lan(); + let (wired, wireless) = (m.wired, m.wireless); + m.shard.send_to(m.now, from_wired, FAR, 9, b"wired").unwrap(); + assert!(datagrams(&mut m).is_empty(), "the wired datagram waits for the gateway's link address"); + // The wireless link asks a retransmission later, so its question is still open when the + // wired link gives the gateway up; nobody answers the wired one, and the wireless one only + // after. + m.now = m.shard.next_deadline().expect("the wired link's resolution is running"); + m.shard.fire(m.now); + datagrams(&mut m); + m.shard.send_to(m.now, from_wireless, FAR, 9, b"wireless").unwrap(); + assert!(datagrams(&mut m).is_empty(), "the wireless datagram waits for the same address on its own link"); + let failed = m.shard.udp_counters().get(toyos_net_udp::Counter::TxUnreachable); + while m.shard.udp_counters().get(toyos_net_udp::Counter::TxUnreachable) == failed { + m.now = m.shard.next_deadline().expect("the wired link's resolution is still running"); + m.shard.fire(m.now); + datagrams(&mut m); + } + assert_eq!(m.shard.udp_counters().get(toyos_net_udp::Counter::TxUnreachable), failed + 1, "the wired datagram alone"); + m.shard.receive(m.now, wireless, &router_reply(LAN_GATEWAY, (WIRELESS_MAC, LAN_WIRELESS))); + assert_eq!(datagrams(&mut m), [(wireless, LAN_WIRELESS)], "the wireless datagram leaves"); + assert!(m.shard.ip().neighbour(wired, LAN_GATEWAY).is_some_and(|n| matches!(n, toyos_net_ip::Nud::Failed)), "the premise: the wired link gave it up"); +} + +/// The ones'-complement sum of `bytes` (RFC 1071). +fn checksum(bytes: &[u8]) -> u16 { + let mut sum: u32 = bytes.chunks(2).map(|c| u32::from(u16::from_be_bytes([c[0], *c.get(1).unwrap_or(&0)]))).sum(); + while sum >> 16 != 0 { + sum = (sum & 0xffff) + (sum >> 16); + } + !(sum as u16) +} + +/// A bare TCP segment from `from` to `to` in a frame from the gateway to `mac`. +fn segment(mac: MacAddr, from: (Ipv4Addr, u16), to: (Ipv4Addr, u16), seq: u32, ack: u32, flags: u8) -> Vec { + let mut tcp = Vec::new(); + tcp.extend_from_slice(&from.1.to_be_bytes()); + tcp.extend_from_slice(&to.1.to_be_bytes()); + tcp.extend_from_slice(&seq.to_be_bytes()); + tcp.extend_from_slice(&ack.to_be_bytes()); + tcp.extend_from_slice(&[0x50, flags, 0xff, 0xff, 0, 0, 0, 0]); + let pseudo = [&from.0.octets()[..], &to.0.octets(), &[0, 6, 0, 20]].concat(); + let check = checksum(&[pseudo, tcp.clone()].concat()); + tcp[16..18].copy_from_slice(&check.to_be_bytes()); + let mut ip = vec![0x45, 0, 0, 40, 0, 0, 0x40, 0, 64, 6, 0, 0]; + ip.extend_from_slice(&from.0.octets()); + ip.extend_from_slice(&to.0.octets()); + let check = checksum(&ip); + ip[10..12].copy_from_slice(&check.to_be_bytes()); + [&mac.0[..], &ROUTER_MAC.0, &[0x08, 0x00], &ip, &tcp].concat() +} + +fn gateway_state(m: &Machine, iface: IfIndex) -> Option<&'static str> { + m.shard.ip().neighbour(iface, LAN_GATEWAY).map(|n| match n { + toyos_net_ip::Nud::Reachable(_) => "reachable", + toyos_net_ip::Nud::Stale(_) => "stale", + toyos_net_ip::Nud::Delay(_) => "delay", + _ => "other", + }) +} + +/// Resolves the gateway on each link, a datagram from each link's socket asking, and leaves +/// both entries stale. +fn both_stale(m: &mut Machine, from_wired: toyos_net_udp::SocketId, from_wireless: toyos_net_udp::SocketId) { + let (wired, wireless) = (m.wired, m.wireless); + for (socket, iface, mine) in [(from_wired, wired, (WIRED_MAC, LAN_WIRED)), (from_wireless, wireless, (WIRELESS_MAC, LAN_WIRELESS))] { + m.shard.send_to(m.now, socket, FAR, 9, b"x").unwrap(); + datagrams(m); + m.shard.receive(m.now, iface, &router_reply(LAN_GATEWAY, mine)); + datagrams(m); + } + let later = m.now.after(std::time::Duration::from_secs(120)); + while let Some(at) = m.shard.next_deadline().filter(|at| *at <= later) { + m.now = at; + m.shard.fire(at); + datagrams(m); + } + m.now = later; + assert_eq!((gateway_state(m, wired), gateway_state(m, wireless)), (Some("stale"), Some("stale")), "the premise: both stale"); +} + +/// A connection's progress confirms the next hop of its own link: where the wired and the +/// wireless link share one gateway address, a flow on the wireless link refreshes the wireless +/// entry and leaves the wired one as it was. +#[test] +fn a_flows_progress_confirms_its_own_links_gateway_where_two_links_share_one() { + let (mut m, from_wired, from_wireless) = one_lan(); + let (wired, wireless) = (m.wired, m.wireless); + // A connection from the wireless link, made while the wired one is down. + m.shard.link_down(m.now, wired).unwrap(); + let flow = m.shard.connect(m.now, None, far(80)).unwrap(); + m.shard.receive(m.now, wireless, &router_reply(LAN_GATEWAY, (WIRELESS_MAC, LAN_WIRELESS))); + let mut syn = None; + for _ in 0..2 { + m.shard.transmit(m.now, usize::MAX, |iface, bytes| { + if iface == wireless && bytes.len() >= 54 && bytes[23] == 6 { + syn = Some((u16::from_be_bytes([bytes[34], bytes[35]]), u32::from_be_bytes([bytes[38], bytes[39], bytes[40], bytes[41]]))); + } + }); + m.shard.receive(m.now, wireless, &router_reply(LAN_GATEWAY, (WIRELESS_MAC, LAN_WIRELESS))); + } + let (port, iss) = syn.expect("its SYN, by the wireless link"); + m.shard.link_up(m.now, wired).unwrap(); + m.shard.receive(m.now, wireless, &segment(WIRELESS_MAC, (FAR, 80), (LAN_WIRELESS, port), 7_000, iss.wrapping_add(1), 0x12)); + datagrams(&mut m); + assert!(m.shard.status(flow).is_ok_and(|s| s.state == toyos_net_tcp::State::Established), "the premise: connected"); + both_stale(&mut m, from_wired, from_wireless); + // Two bytes sent and acknowledged: SND.UNA advances, and the advice is the wireless link's. + assert_eq!(m.shard.send(m.now, flow, b"hi"), Ok(2)); + datagrams(&mut m); + m.shard.receive(m.now, wireless, &segment(WIRELESS_MAC, (FAR, 80), (LAN_WIRELESS, port), 7_001, iss.wrapping_add(3), 0x10)); + assert_eq!(gateway_state(&m, wireless), Some("reachable"), "the flow's own gateway entry, confirmed"); + assert_eq!(gateway_state(&m, wired), Some("stale"), "the other link's entry for the same address, untouched"); +} diff --git a/system/netstack/shard/tests/log.rs b/system/netstack/shard/tests/log.rs index 64dc65348d2..fa426d01ddc 100644 --- a/system/netstack/shard/tests/log.rs +++ b/system/netstack/shard/tests/log.rs @@ -30,13 +30,13 @@ fn refusals(events: &[Event]) -> Vec<(Refusal, u64)> { fn s_ip_mod_001_shard_one_line_per_rule_per_10_s() { let (mut net, a, _) = segment(); let start = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; shard.drain_events().for_each(drop); let frame = from_b(&hex(V_IP_LSRR)); for n in 0..100 { - shard.receive(start.after(Duration::from_millis(n * 10)), &frame); + shard.receive(start.after(Duration::from_millis(n * 10)), iface, &frame); } - shard.receive(start.after(Duration::from_millis(10_001)), &frame); + shard.receive(start.after(Duration::from_millis(10_001)), iface, &frame); let lines = refusals(&shard.drain_events().collect::>()); let rules: Vec<(toyos_net_ip::Counter, Peer, u64)> = lines .iter() @@ -54,13 +54,13 @@ fn s_ip_mod_001_shard_one_line_per_rule_per_10_s() { fn s_mod_005_shard_one_line_per_rule_per_10_s() { let (mut net, a, _) = segment(); let start = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; shard.drain_events().for_each(drop); let syn_fin = |n: u8| from_b(&segment_bytes((Ipv4Addr::new(192, 0, 2, 10 + n), 1234), (A, 80), 5_000, 0x03)); for n in 0..50 { - shard.receive(start.after(Duration::from_millis(u64::from(n))), &syn_fin(n)); + shard.receive(start.after(Duration::from_millis(u64::from(n))), iface, &syn_fin(n)); } - shard.receive(start.after(Duration::from_millis(10_000)), &syn_fin(50)); + shard.receive(start.after(Duration::from_millis(10_000)), iface, &syn_fin(50)); let lines: Vec<(Ipv4Addr, u16, u64)> = refusals(&shard.drain_events().collect::>()) .into_iter() .map(|(r, n)| match r { diff --git a/system/netstack/shard/tests/udp.rs b/system/netstack/shard/tests/udp.rs index 846a2aa746e..95fb44527ed 100644 --- a/system/netstack/shard/tests/udp.rs +++ b/system/netstack/shard/tests/udp.rs @@ -9,7 +9,7 @@ use std::net::Ipv4Addr; use std::time::Duration; use common::*; -use toyos_net_shard::{Event, Refusal, Shard}; +use toyos_net_shard::{Event, IfIndex, Refusal, Shard}; use toyos_net_testnet::Fate; use toyos_net_ip::Nud; use toyos_net_udp::{Counter, Error, SocketError}; @@ -42,7 +42,7 @@ fn s_udp_us_046_shard_a_port_unreachable_refuses_the_connected_socket_once() { net.advance(Duration::from_millis(1)); let datagram = net.wire()[start..].iter().find(|c| c.from == a && name(&c.frame).starts_with("IPv4 Udp")).expect("the datagram left").frame.clone(); let now = net.now(); - net.nodes[a].shard.receive(now, &icmp_about(&datagram, 3, 3, [0; 4])); + net.nodes[a].receive(now, &icmp_about(&datagram, 3, 3, [0; 4])); let mut buf = [0u8; 16]; assert_eq!(net.nodes[a].shard.recv_from(socket, &mut buf), Err(Error::Failed(SocketError::Refused))); assert_eq!(net.nodes[a].shard.recv_from(socket, &mut buf), Ok(None)); @@ -85,8 +85,8 @@ fn b_resolved() -> (toyos_net_testnet::Net, usize, toyos_net_udp::SocketId) { let socket = net.nodes[a].shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5000)), || 0).unwrap(); net.nodes[a].shard.send_to(now, socket, B, 9, b"hello").unwrap(); net.advance(Duration::from_millis(1)); - let shard = &net.nodes[a].shard; - assert!(shard.ip().neighbour(shard.iface(), B).is_some_and(|entry| entry.mac().is_some()) && shard.ip().neighbour(shard.iface(), C).is_none()); + let (shard, iface) = (&net.nodes[a].shard, net.nodes[a].iface); + assert!(shard.ip().neighbour(iface, B).is_some_and(|entry| entry.mac().is_some()) && shard.ip().neighbour(iface, C).is_none()); (net, a, socket) } @@ -94,7 +94,7 @@ fn b_resolved() -> (toyos_net_testnet::Net, usize, toyos_net_udp::SocketId) { /// destination and payload, any other frame by its name. fn left(shard: &mut Shard, now: Instant) -> Vec { let mut left = Vec::new(); - shard.transmit(now, usize::MAX, |bytes| { + shard.transmit(now, usize::MAX, |_, bytes| { let frame = Frame::parse(bytes).expect("a frame"); let udp = Ipv4Packet::parse(frame.body()).ok().and_then(|ip| UdpDatagram::parse(&ip).ok().map(|udp| format!("UDP {} {}", ip.destination(), String::from_utf8_lossy(udp.payload())))); left.push(udp.unwrap_or_else(|| name(bytes))); @@ -103,8 +103,8 @@ fn left(shard: &mut Shard, now: Instant) -> Vec { } /// How many datagrams [ip] holds for C while it asks for its link address. -fn held_by_ip(shard: &Shard) -> Option { - match shard.ip().neighbour(shard.iface(), C) { +fn held_by_ip(shard: &Shard, iface: IfIndex) -> Option { + match shard.ip().neighbour(iface, C) { Some(Nud::Incomplete(asking)) => Some(asking.pending.queued()), _ => None, } @@ -117,14 +117,14 @@ fn held_by_ip(shard: &Shard) -> Option { fn s_udp_us_026_shard_a_datagram_waits_in_its_socket_for_its_next_hop_and_holds_back_no_other() { let (mut net, a, socket) = b_resolved(); let now = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; for (destination, data) in [(C, "c1"), (B, "b1"), (C, "c2")] { shard.send_to(now, socket, destination, 9, data.as_bytes()).unwrap(); } assert_eq!(left(shard, now), ["UDP 192.0.2.2 b1", "ARP request 192.0.2.3"]); - assert_eq!(held_by_ip(shard), Some(0), "[ip] holds no sender's datagram"); + assert_eq!(held_by_ip(shard, iface), Some(0), "[ip] holds no sender's datagram"); assert_eq!(left(shard, now), [""; 0], "what waits asks for nothing again"); - shard.receive(now, &c_answers()); + shard.receive(now, iface, &c_answers()); assert_eq!(left(shard, now), ["UDP 192.0.2.3 c1", "UDP 192.0.2.3 c2"]); } @@ -135,7 +135,7 @@ fn s_udp_us_026_shard_a_datagram_waits_in_its_socket_for_its_next_hop_and_holds_ fn s_udp_us_025_shard_what_waits_for_a_next_hop_is_bounded_by_its_sockets_queue() { let (mut net, a, socket) = b_resolved(); let now = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; let other = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5002)), || 0).unwrap(); let per = toyos_net_udp::limits::TX_DATAGRAMS; for n in 0..per { @@ -146,8 +146,8 @@ fn s_udp_us_025_shard_what_waits_for_a_next_hop_is_bounded_by_its_sockets_queue( for destination in [C, B] { assert_eq!(shard.send_to(now, socket, destination, 9, b"x"), Err(Error::Refused(Counter::TxQueueFull)), "{destination}"); } - assert_eq!(held_by_ip(shard), Some(0)); - shard.receive(now, &c_answers()); + assert_eq!(held_by_ip(shard, iface), Some(0)); + shard.receive(now, iface, &c_answers()); let out = left(shard, now); for (sender, tag) in [(socket, 's'), (other, 'o')] { let of: Vec<&String> = out.iter().filter(|frame| frame.starts_with(&format!("UDP 192.0.2.3 {tag}"))).collect(); @@ -176,7 +176,7 @@ fn s_udp_us_027_shard_a_failed_next_hop_drops_what_waited_for_it_and_tells_no_un assert_eq!((shard.udp_counters().get(Counter::TxUnreachable), shard.ip().counters().get(toyos_net_ip::Counter::NbFailed)), (2, 1)); assert_eq!(shard.recv_from(socket, &mut [0; 16]), Ok(None)); let now = net.now(); - net.nodes[a].shard.receive(now, &c_answers()); + net.nodes[a].receive(now, &c_answers()); net.advance(Duration::from_millis(1)); assert!(net.wire()[start..].iter().all(|carried| !name(&carried.frame).starts_with("IPv4 Udp")), "{}", dump(&net, start)); } @@ -188,13 +188,13 @@ fn s_udp_us_027_shard_a_failed_next_hop_drops_what_waited_for_it_and_tells_no_un fn s_udp_us_024_shard_a_waiting_datagram_whose_route_goes_is_dropped() { let (mut net, a, socket) = b_resolved(); let now = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; shard.send_to(now, socket, C, 9, b"c1").unwrap(); assert_eq!(left(shard, now), ["ARP request 192.0.2.3"]); - shard.link_down(now).unwrap(); + shard.link_down(now, iface).unwrap(); assert_eq!((left(shard, now), shard.udp_counters().get(Counter::TxUnreachable), shard.ip().counters().get(toyos_net_ip::Counter::RouteNoSourceAddress)), (vec![], 0, 1)); - shard.link_up(now).unwrap(); - shard.receive(now, &c_answers()); + shard.link_up(now, iface).unwrap(); + shard.receive(now, iface, &c_answers()); assert!(left(shard, now).iter().all(|frame| !frame.starts_with("UDP"))); } @@ -205,14 +205,14 @@ fn s_udp_us_024_shard_a_waiting_datagram_whose_route_goes_is_dropped() { fn a_waiting_datagram_whose_link_returns_before_an_opportunity_waits_anew_and_leaves() { let (mut net, a, socket) = b_resolved(); let now = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; shard.send_to(now, socket, C, 9, b"c1").unwrap(); assert_eq!(left(shard, now), ["ARP request 192.0.2.3"]); - shard.link_down(now).unwrap(); - shard.link_up(now).unwrap(); + shard.link_down(now, iface).unwrap(); + shard.link_up(now, iface).unwrap(); assert_eq!(left(shard, now), ["ARP request 192.0.2.1", "ARP request 192.0.2.3"], "the address announced, and the hop asked for anew"); assert_eq!((shard.udp_counters().get(Counter::TxUnreachable), shard.ip().counters().get(toyos_net_ip::Counter::RouteNoSourceAddress)), (0, 0)); - shard.receive(now, &c_answers()); + shard.receive(now, iface, &c_answers()); assert_eq!(left(shard, now), ["UDP 192.0.2.3 c1"]); } @@ -227,7 +227,7 @@ const ROUTER_IN_25: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 126); /// and, of a UDP datagram, its destination and source port; any other frame by its name. fn opportunity(shard: &mut Shard, now: Instant) -> Vec<(MacAddr, String)> { let mut left = Vec::new(); - shard.transmit(now, usize::MAX, |bytes| { + shard.transmit(now, usize::MAX, |_, bytes| { let frame = Frame::parse(bytes).expect("a frame"); let udp = Ipv4Packet::parse(frame.body()).ok().and_then(|ip| UdpDatagram::parse(&ip).ok().map(|udp| format!("UDP {} from port {}", ip.destination(), udp.source_port().map_or(0, |p| p.get())))); left.push((frame.destination(), udp.unwrap_or_else(|| name(bytes)))); @@ -237,11 +237,11 @@ fn opportunity(shard: &mut Shard, now: Instant) -> Vec<(MacAddr, String)> { /// [ip] counted `count` datagrams refused a link broadcast, and the shard's one line for the /// rule names `destination`. -fn refused_a_broadcast(shard: &mut Shard, destination: Ipv4Addr, count: u64) { +fn refused_a_broadcast(shard: &mut Shard, iface: IfIndex, destination: Ipv4Addr, count: u64) { let rule = toyos_net_ip::Counter::IpBroadcastNotPermitted; assert_eq!(shard.ip().counters().get(rule), count); let lines: Vec = shard.drain_events().filter(|event| matches!(event, Event::Refused { .. })).collect(); - let refusal = toyos_net_ip::Refusal { rule, iface: shard.iface(), peer: toyos_net_ip::Peer::Ip(destination) }; + let refusal = toyos_net_ip::Refusal { rule, iface, peer: toyos_net_ip::Peer::Ip(destination) }; assert_eq!(lines, [Event::Refused { refusal: Refusal::Ip(refusal), suppressed: 0 }]); } @@ -256,16 +256,16 @@ fn s_udp_us_021_shard_a_datagram_accepted_for_a_host_is_no_broadcast_after_a_ren for closed in [false, true] { let (mut net, a, _) = segment(); let now = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; shard.drain_events().for_each(drop); let socket = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5000)), || 0).unwrap(); assert_eq!(shard.send_to(now, socket, EDGE_OF_25, 5001, b"hi"), Ok(()), "one host of the /24"); if closed { shard.udp_close(now, socket).unwrap(); } - assert_eq!(shard.add_address(now, A, 25), Ok(()), "the renewal"); + assert_eq!(shard.add_address(now, iface, A, 25), Ok(()), "the renewal"); assert_eq!(opportunity(shard, now), [], "closed: {closed}"); - refused_a_broadcast(shard, EDGE_OF_25, 1); + refused_a_broadcast(shard, iface, EDGE_OF_25, 1); assert_eq!(opportunity(shard, now), [], "and it is gone, not waiting"); } } @@ -276,15 +276,15 @@ fn s_udp_us_021_shard_a_datagram_accepted_for_a_host_is_no_broadcast_after_a_ren fn s_udp_us_021_shard_a_datagram_accepted_for_the_router_is_no_broadcast_after_a_renewal_widens_the_prefix() { let (mut net, a, _) = segment(); let now = net.now(); - let shard = &mut net.nodes[a].shard; - shard.add_address(now, A, 25).unwrap(); - shard.set_gateways(now, &[ROUTER_IN_25]).unwrap(); + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; + shard.add_address(now, iface, A, 25).unwrap(); + shard.set_gateways(now, iface, &[ROUTER_IN_25]).unwrap(); shard.drain_events().for_each(drop); let socket = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5000)), || 0).unwrap(); assert_eq!(shard.send_to(now, socket, EDGE_OF_24, 5001, b"hi"), Ok(()), "a host behind the router"); - assert_eq!(shard.add_address(now, A, 24), Ok(()), "the renewal"); + assert_eq!(shard.add_address(now, iface, A, 24), Ok(()), "the renewal"); assert_eq!(opportunity(shard, now), []); - refused_a_broadcast(shard, EDGE_OF_24, 1); + refused_a_broadcast(shard, iface, EDGE_OF_24, 1); } // US-21 for a datagram that waited: accepted for one host of the /24 whose link address nobody @@ -295,7 +295,7 @@ fn s_udp_us_021_shard_a_datagram_accepted_for_the_router_is_no_broadcast_after_a fn s_udp_us_021_shard_a_datagram_that_waited_for_a_host_is_asked_its_route_again_and_keeps_its_permission() { let (mut net, a, _) = segment(); let now = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; shard.drain_events().for_each(drop); let permitted = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5000)), || 0).unwrap(); let unpermitted = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5002)), || 0).unwrap(); @@ -305,9 +305,9 @@ fn s_udp_us_021_shard_a_datagram_that_waited_for_a_host_is_asked_its_route_again } shard.udp_set_broadcast(permitted, false).unwrap(); assert_eq!(opportunity(shard, now), [(MacAddr::BROADCAST, "ARP request 192.0.2.127".to_string())], "both wait for the host"); - assert_eq!(shard.add_address(now, A, 25), Ok(()), "the renewal"); + assert_eq!(shard.add_address(now, iface, A, 25), Ok(()), "the renewal"); assert_eq!(opportunity(shard, now), [(MacAddr::BROADCAST, "UDP 192.0.2.127 from port 5000".to_string())]); - refused_a_broadcast(shard, EDGE_OF_25, 1); + refused_a_broadcast(shard, iface, EDGE_OF_25, 1); } // The permission is the one the socket held at the call (POSIX's `SO_BROADCAST` is read by the @@ -317,7 +317,7 @@ fn s_udp_us_021_shard_a_datagram_that_waited_for_a_host_is_asked_its_route_again fn s_udp_us_021_shard_a_datagram_carries_the_permission_of_its_call() { let (mut net, a, _) = segment(); let now = net.now(); - let shard = &mut net.nodes[a].shard; + let (shard, iface) = { let node = &mut net.nodes[a]; (&mut node.shard, node.iface) }; shard.drain_events().for_each(drop); let permitted = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5000)), || 0).unwrap(); let unpermitted = shard.bind(Ipv4Addr::UNSPECIFIED, Some(port(5002)), || 0).unwrap(); @@ -327,7 +327,7 @@ fn s_udp_us_021_shard_a_datagram_carries_the_permission_of_its_call() { } shard.udp_set_broadcast(permitted, false).unwrap(); shard.udp_set_broadcast(unpermitted, true).unwrap(); - assert_eq!(shard.add_address(now, A, 25), Ok(()), "the renewal"); + assert_eq!(shard.add_address(now, iface, A, 25), Ok(()), "the renewal"); assert_eq!(opportunity(shard, now), [(MacAddr::BROADCAST, "UDP 192.0.2.127 from port 5000".to_string())]); - refused_a_broadcast(shard, EDGE_OF_25, 1); + refused_a_broadcast(shard, iface, EDGE_OF_25, 1); } diff --git a/system/netstack/src/link.rs b/system/netstack/src/link.rs index 0ab6bd16d40..30fdb000755 100644 --- a/system/netstack/src/link.rs +++ b/system/netstack/src/link.rs @@ -16,7 +16,7 @@ use toyos_link::layout::{RX_DEPTH, SLOT_BYTES}; use toyos_link::region::Region; use toyos_link::state::{Counters, Group, Link}; use toyos_link::wire::{self, Hello, Refusal}; -use toyos_net_node::Node; +use toyos_net_node::{Credit, IfIndex, Node}; use toyos_net_wire::ethernet::{IndividualMac, MacAddr}; use toyos_net_wire::Instant; @@ -96,7 +96,10 @@ pub struct Session { region: Region, end: Stack, hello: Hello, - mac: IndividualMac, + /// The node's interface for the link. + iface: IfIndex, + /// Frames the stack handed the link: its own count, not the driver's. + sent: u64, /// The link as the node was last told it. said: Link, frame: Box<[u8; SLOT_BYTES]>, @@ -106,24 +109,19 @@ pub struct Session { } impl Session { - /// Open the link `hello` asked for over `region`, and say so to the - /// driver; `None` for a driver gone before it heard. - pub fn open(hello: Hello, mac: IndividualMac, conn: Connection, region: Region, groups: &[Group]) -> Option { + /// Open the link `hello` asked for over `region` as the node's `iface`, + /// and say so to the driver; `None` for a driver gone before it heard. + pub fn open(hello: Hello, iface: IfIndex, conn: Connection, region: Region, groups: &[Group]) -> Option { let mut end = Stack::new(region.words()); for group in groups { assert!(end.join(region.words(), *group), "netstack: more groups than a link holds"); } conn.try_send_bytes(wire::MSG_OPENED, &[]).ok()?; - Some(Self { conn, region, end, hello, mac, said: Link::Down, frame: Box::new([0; SLOT_BYTES]), asked: false, ring: true }) + Some(Self { conn, region, end, hello, iface, sent: 0, said: Link::Down, frame: Box::new([0; SLOT_BYTES]), asked: false, ring: true }) } - pub fn hello(&self) -> &Hello { - &self.hello - } - - /// The driver's address, as the node takes one. - pub fn mac(&self) -> IndividualMac { - self.mac + pub fn iface(&self) -> IfIndex { + self.iface } /// The link's end if its driver has hung up, read now. @@ -147,13 +145,14 @@ impl Session { self.hear()?; let link = self.end.link(self.region.words()).map_err(Ended::Fault)?; if link.is_up() != self.said.is_up() { - node.link(now, link.is_up(), draw); + node.link(now, self.iface, link.is_up(), draw); } self.said = link; let (words, bytes, end, frame) = (self.region.words(), self.region.bytes(), &mut self.end, &mut self.frame); let mut fault = None; let owed = node.receive( now, + self.iface, RX_DEPTH as usize, |sink| match end.received(words) { Ok(Some(span)) => { @@ -177,36 +176,26 @@ impl Session { } } - /// Offer the node the transmit ring's room until it has no frame left or - /// the ring no room, and ring the driver for what was sent. - pub fn transmit(&mut self, node: &mut Node, now: Instant, draw: fn() -> u32) -> Result<(), Ended> { + /// The transmit ring's room, armed to be rung for when there is none. + fn room(&mut self) -> Result { let words = self.region.words(); - loop { - let room = match self.end.room(words).map_err(Ended::Fault)? { - 0 => self.end.want_room(words).map_err(Ended::Fault)?, - room => room, - } as usize; - let (bytes, end) = (self.region.bytes(), &mut self.end); - let mut fault = None; - let sent = node.transmit( - now, - room, - |frame| { - bytes.write(end.slot().offset, frame); - if let Err(why) = end.send(words, frame.len()) { - fault.get_or_insert(why); - } - }, - draw, - ); - if let Some(why) = fault { - return Err(Ended::Fault(why)); - } - self.ring |= self.end.publish(words); - if room == 0 || sent < room { - return Ok(()); - } - } + Ok(match self.end.room(words)? { + 0 => self.end.want_room(words)?, + room => room, + } as usize) + } + + /// One frame into the transmit ring's next slot. + fn send(&mut self, frame: &[u8]) -> Result<(), Fault> { + self.region.bytes().write(self.end.slot().offset, frame); + self.end.send(self.region.words(), frame.len())?; + self.sent = self.sent.saturating_add(1); + Ok(()) + } + + /// What was sent reaches the driver, which is rung for it. + fn publish(&mut self) { + self.ring |= self.end.publish(self.region.words()); } /// Ask the driver for its counters, which [`Self::counters`] reads once @@ -261,21 +250,87 @@ impl Session { /// **An unreported link is `unreported`, not `up`**: the device tells the /// stack nothing about one, and the stack serving as though it were up is /// its assumption rather than something it measured. - pub fn inspect(&self, counters: Option<&Counters>, snap: &mut Snapshot) { + pub fn inspect(&self, at: &str, counters: Option<&Counters>, snap: &mut Snapshot) { let m = self.hello.mac.bytes(); - snap.put("mac", format!("{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", m[0], m[1], m[2], m[3], m[4], m[5])); - snap.put("driver", self.hello.label.as_str()); + snap.put(&format!("{at}.mac"), format!("{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", m[0], m[1], m[2], m[3], m[4], m[5])); + snap.put(&format!("{at}.driver"), self.hello.label.as_str()); + snap.put( + &format!("{at}.medium"), + match self.hello.medium { + wire::Medium::Wired => "wired", + wire::Medium::Wireless => "wireless", + }, + ); + snap.put(&format!("{at}.sent"), self.sent); match self.said { - Link::Down => snap.put("link.state", "down"), - Link::Unreported => snap.put("link.state", "unreported"), + Link::Down => snap.put(&format!("{at}.state"), "down"), + Link::Unreported => snap.put(&format!("{at}.state"), "unreported"), + Link::UpWithoutSpeed => snap.put(&format!("{at}.state"), "up"), Link::Up { mbps, full_duplex } => { - snap.put("link.state", "up"); - snap.put("link.speed_mbps", mbps); - snap.put("link.duplex", if full_duplex { "full" } else { "half" }); + snap.put(&format!("{at}.state"), "up"); + snap.put(&format!("{at}.speed_mbps"), mbps); + snap.put(&format!("{at}.duplex"), if full_duplex { "full" } else { "half" }); } } for (counter, value) in counters.into_iter().flat_map(Counters::iter) { - snap.put(counter.name(), value); + snap.put(&format!("{at}.{}", counter.name()), value); + } + } +} + +/// Each open link's transmit room in one opportunity, a frame of it spent as +/// the node hands the link one. +struct Rooms(Vec<(IfIndex, usize)>); + +impl Credit for Rooms { + fn room(&self, iface: IfIndex) -> bool { + self.0.iter().any(|&(at, room)| at == iface && room > 0) + } + fn spend(&mut self, iface: IfIndex) { + if let Some((_, room)) = self.0.iter_mut().find(|(at, _)| *at == iface) { + *room = room.saturating_sub(1); + } + } +} + +/// Offer the node every open link's transmit room until it has no frame left +/// or no link room, and ring each driver for what was sent. Returns each link +/// whose driver wrote what no driver writes, which is to end. +pub fn transmit(sessions: &mut [&mut Session], node: &mut Node, now: Instant, draw: fn() -> u32) -> Vec<(IfIndex, Fault)> { + let mut faults: Vec<(IfIndex, Fault)> = Vec::new(); + loop { + let mut rooms = Rooms(Vec::new()); + for session in sessions.iter_mut() { + if faults.iter().any(|(at, _)| *at == session.iface) { + continue; + } + match session.room() { + Ok(room) => rooms.0.push((session.iface, room)), + Err(why) => faults.push((session.iface, why)), + } + } + let offered: usize = rooms.0.iter().map(|(_, room)| room).sum(); + if offered == 0 { + return faults; + } + let sent = node.transmit( + now, + &mut rooms, + |iface, frame| { + let Some(session) = sessions.iter_mut().find(|s| s.iface == iface) else { unreachable!("the node sent by a link netstack does not hold") }; + if let Err(why) = session.send(frame) + && !faults.iter().any(|(at, _)| *at == iface) + { + faults.push((iface, why)); + } + }, + draw, + ); + for session in sessions.iter_mut() { + session.publish(); + } + if sent < offered { + return faults; } } } diff --git a/system/netstack/src/main.rs b/system/netstack/src/main.rs index a4efe2052d4..65702ac6162 100644 --- a/system/netstack/src/main.rs +++ b/system/netstack/src/main.rs @@ -1,5 +1,5 @@ -//! The network server: ToyOS's own stack (`toyos-net-node`) over the link a -//! NIC driver brings (`toyos-link`), served to programs over `toyos::net`'s +//! The network server: ToyOS's own stack (`toyos-net-node`) over every link +//! a NIC driver brings (`toyos-link`), served to programs over `toyos::net`'s //! protocol. //! //! **This program holds no protocol and drives no device.** Every byte off the @@ -10,18 +10,17 @@ //! holders are rung on (`bells`), the clients' connections (`client`) and //! their requests (`serve`). //! -//! **One link, until netstack runs several** -//! (`issues/netstack-drives-one-card.md`): the node is made with the first -//! driver's address, so nothing is served before that driver's hello, and a -//! machine none says hello on within [`LINK_BOUND`] has no network: netstack -//! leaves, as a client then hears. A link that ends leaves every connection -//! to the node, and only a hello with that address is opened again. +//! **Every link a driver brings**, up to [`MAX_LINKS`]: each is an interface +//! of the node's, ranked wired before wireless and then by arrival, with its +//! own lease. The node serves from the start, before any link. A link that +//! ends keeps its interface, down, and a hello with its address is opened +//! on it again; a hello with the address of a link that is open is refused. //! -//! **One pass**: the link's state and its received frames, as one batch, go to the node, then -//! every deadline that is due, then the node is offered the link's transmit -//! room until it has no frame left or the link no room; what the node has to +//! **One pass**: each link's state and its received frames, as one batch, go to the node, then +//! every deadline that is due, then the node is offered every link's transmit +//! room until it has no frame left or no link room; what the node has to //! say is written to the log and to the clients that waited for it; and the -//! loop sleeps on the kernel until the link, a client, a watched handle or the +//! loop sleeps on the kernel until a link, a client, a watched handle or the //! node's next deadline wakes it. A request is carried out in the pass that //! read it and its frames leave in the next, which follows at once. //! @@ -44,9 +43,9 @@ use toyos_abi::syscall::PIPE_CAPACITY; use toyos_dhcp::{HostName, Lease}; use toyos_inspect::Snapshot; use toyos_link::state::{Counters, Group}; -use toyos_link::wire::{Hello, Refusal}; +use toyos_link::wire::{Hello, Medium, Refusal}; use toyos_net_ip::Nud; -use toyos_net_node::Node; +use toyos_net_node::{IfIndex, Node, Rank}; use toyos_net_shard::{Config, Secrets}; use toyos_net_wire::ethernet::IndividualMac; use toyos_net_wire::Instant; @@ -61,26 +60,25 @@ mod watches; use client::{Client, ClientRx, HANDSHAKE_TIMEOUT, MAX_PENDING_CONNS, PendingConn, Request}; use link::{Ended, Heard, Pending, Session}; -/// How long netstack waits for its first link before it leaves as a machine -/// with no network. Policy: a driver says hello once its device is up, which -/// is seconds of this; a machine with no NIC driver waits it out, and its -/// clients with it. -const LINK_BOUND: Duration = Duration::from_secs(30); +/// The most links netstack drives. Each is an interface of the node's for +/// the boot, kept for its driver's return; a machine has a handful of NICs. +const MAX_LINKS: u32 = 8; -/// How long an `inspect` waits for the driver's counters. The driver answers +/// How long an `inspect` waits for the drivers' counters. A driver answers /// in the pass the ask's doorbell starts; one that does not is answered for /// without them. const INSPECT_BOUND: Duration = Duration::from_secs(1); -/// Drivers' connections that have not said hello, watched at once. -const MAX_PENDING_LINKS: u32 = 2; +/// Drivers' connections that have not said hello, watched at once: every +/// link's driver may be starting together. +const MAX_PENDING_LINKS: u32 = MAX_LINKS; /// The name this machine asks its network to record for it, and answers to on /// it as `.local` once no other host does. One name, because there is /// one machine. pub const HOSTNAME: &str = "toyos-t14"; -/// How long this machine waits for its first lease before saying it has none. +/// How long this machine waits for its first lease, on any link, before saying it has none. /// It bounds the report, never the client: the node asks for the life of the /// boot and a lease that lands later is applied like any other. const LEASE_BOUND: Duration = Duration::from_millis(toyos_tco::LEASE_BOUND_MS); @@ -129,9 +127,9 @@ const PLACE_BYTES: u64 = { /// this in favour of a kernel memory limit, not in favour of a bigger number. const PLACE_BUDGET_SHARE: u64 = 8; -/// Watches that are no place's: the service's two acceptors, the link, and +/// Watches that are no place's: the service's two acceptors, the links, and /// the drivers' connections that have not said hello. -const FIXED_WATCHES: u32 = 3 + MAX_PENDING_LINKS; +const FIXED_WATCHES: u32 = 2 + MAX_LINKS + MAX_PENDING_LINKS; /// The most places one poller can watch: each place is one watched handle, a /// holder's connection of a stream, a listener or a datagram socket, asked @@ -197,30 +195,36 @@ impl Said { } } -/// What the boot's log has said of the lease, and still owes. +/// What the boot's log has said of each link's lease, and still owes. struct Leases { began: Wall, - said: Option, - /// Whether this boot has settled the question once: a lease landed, or - /// the bound passed with none. + said: Vec<(IfIndex, Said)>, + /// Whether this boot has settled the question once: a lease landed on a + /// link, or the bound passed with none. settled: bool, } impl Leases { - /// Says what changed of the lease the node holds, and answers whether - /// this machine's address question has just been settled. - fn pass(&mut self, held: Option<&Lease>) -> bool { - let held = held.map(Said::of); - if held != self.said { + /// Says what changed of each link's lease, `label` naming the link, and + /// answers whether this machine's address question has just been + /// settled. + fn pass(&mut self, node: &Node, label: impl Fn(IfIndex) -> String) -> bool { + for link in node.links() { + let held = node.lease(link).map(Said::of); + let at = self.said.iter().position(|(on, _)| *on == link); + if held.as_ref() == at.map(|at| &self.said[at].1) { + continue; + } match &held { // One record carrying every field the lease decided: a boot // read off a stick or a stream has this line and nothing else // to say what this machine's network was. Some(lease) => say!( - "netstack: DHCP: lease {}/{} from {}, gateway {}, dns [{}], {} ms after \ + "netstack: DHCP: lease {}/{} on {}, from {}, gateway {}, dns [{}], {} ms after \ netstack came up", lease.address, lease.prefix_len, + label(link), lease.server, match lease.router { Some(router) => router.to_string(), @@ -229,20 +233,25 @@ impl Leases { lease.dns(), self.began.elapsed().as_millis(), ), - None => say!("netstack: DHCP: the lease is gone; this machine has no address"), + None => say!("netstack: DHCP: the lease on {} is gone; that link has no address", label(link)), + } + match (held, at) { + (Some(held), Some(at)) => self.said[at].1 = held, + (Some(held), None) => self.said.push((link, held)), + (None, Some(at)) => drop(self.said.remove(at)), + (None, None) => {} } - self.said = held; } if self.settled { return false; } - if self.said.is_none() { + if self.said.is_empty() { if self.began.elapsed() < LEASE_BOUND { return false; } say!( - "netstack: DHCP: no lease as {} in {} s; this machine has no address and every \ - connect through it is refused", + "netstack: DHCP: no lease as {} on any link in {} s; this machine has no address \ + and every connect through it is refused", HOSTNAME, self.began.elapsed().as_secs(), ); @@ -256,23 +265,22 @@ impl Leases { (!self.settled).then(|| LEASE_BOUND.saturating_sub(self.began.elapsed())) } - /// The lease as `inspect` reads it, and the router's entry in the - /// stack's neighbour table where the lease names one. - fn inspect(&self, node: &Node, snap: &mut Snapshot) { - let Some(held) = &self.said else { - snap.put("lease.held", false); + /// `link`'s lease as `inspect` reads it, under `at`, and the router's + /// entry in the stack's neighbour table where the lease names one. + fn inspect(&self, node: &Node, link: IfIndex, at: &str, snap: &mut Snapshot) { + let Some((_, held)) = self.said.iter().find(|(on, _)| *on == link) else { + snap.put(&format!("{at}.lease.held"), false); return; }; - snap.put("lease.held", true); - snap.put("lease.address", format!("{}/{}", held.address, held.prefix_len)); - snap.put("lease.server", held.server.to_string()); - snap.put("lease.dns", held.dns()); + snap.put(&format!("{at}.lease.held"), true); + snap.put(&format!("{at}.lease.address"), format!("{}/{}", held.address, held.prefix_len)); + snap.put(&format!("{at}.lease.server"), held.server.to_string()); + snap.put(&format!("{at}.lease.dns"), held.dns()); let Some(router) = held.router else { return }; - snap.put("lease.router", router.to_string()); - let shard = node.shard(); + snap.put(&format!("{at}.lease.router"), router.to_string()); snap.put( - "neighbour.router", - match shard.ip().neighbour(shard.iface(), router) { + &format!("{at}.neighbour.router"), + match node.shard().ip().neighbour(link, router) { None => "none", Some(Nud::Incomplete(_)) => "incomplete", Some(Nud::Reachable(_)) => "reachable", @@ -286,27 +294,63 @@ impl Leases { } } +/// A link netstack has opened this boot: its interface, kept for its driver's +/// return, and its session while the link is open. +struct Held { + iface: IfIndex, + mac: IndividualMac, + label: String, + session: Option, + /// Its last link ended for what its driver wrote: its next hello is + /// refused for it. + faulted: bool, +} + +/// Where a link stands among the others: wired before wireless. +fn rank(medium: Medium) -> Rank { + match medium { + Medium::Wired => Rank(0), + Medium::Wireless => Rank(1), + } +} + +/// `link.`, the `n`th link opened this boot, in `inspect`'s paths. +fn path(n: usize) -> String { + format!("link.{n}") +} + /// Answer `inspect` with what this pass knows, in one non-blocking write, and /// let the connection close as every other answer does. -fn answer_inspect(request: &Request, link: Option<(&Session, Option<&Counters>)>, leases: &Leases, sockets: &serve::Sockets, node: &Node) { +fn answer_inspect(request: &Request, links: &[Held], counters: &[(IfIndex, Counters)], leases: &Leases, sockets: &serve::Sockets, node: &Node) { let mut snap = Snapshot::new(toyos_inspect::NET); - match link { - Some((link, counters)) => link.inspect(counters, &mut snap), - None => snap.put("link.state", "none"), + snap.put("links", links.len() as u32); + for (n, held) in links.iter().enumerate() { + let at = path(n); + match &held.session { + Some(session) => session.inspect(&at, counters.iter().find(|(on, _)| *on == held.iface).map(|(_, c)| c), &mut snap), + None => { + snap.put(&format!("{at}.driver"), held.label.as_str()); + snap.put(&format!("{at}.state"), "ended"); + } + } + leases.inspect(node, held.iface, &at, &mut snap); + if node.asked() == Some(held.iface) { + snap.put("dns.link", n as u32); + } } - leases.inspect(node, &mut snap); sockets.inspect(node, &mut snap); let encoded = snap.encode().unwrap_or_else(|why| panic!("netstack: its snapshot: {why}")); request.client.snapshot(&encoded); } -/// Open the link `asked` asks for, and say so. -fn open(asked: Asked, groups: &[Group]) -> Option { +/// Open the link `asked` asks for on the node's `iface`, the `n`th link of +/// the boot, and say so. +fn open(asked: Asked, iface: IfIndex, n: usize, groups: &[Group]) -> Option { let label = asked.hello.label.clone(); - let opened = Session::open(asked.hello, asked.mac, asked.conn, asked.region, groups); + let opened = Session::open(asked.hello, iface, asked.conn, asked.region, groups); match &opened { - Some(_) => say!("netstack: {label}'s link is open"), - None => say!("netstack: {label} went before its link was opened"), + Some(_) => say!("netstack: {label}'s link is open, as link {n}"), + None => say!("netstack: {label} went before its link {n} was opened"), } opened } @@ -339,18 +383,56 @@ fn hear_links(pending: &mut Vec) -> Vec { asked } -/// End `link` for `why`; the node hears a change, not a state. -fn end_link(link: &mut Option, why: Ended, node: &mut Node, now: Instant, faulted: &mut bool) { - let Some(session) = link.take() else { return }; +/// End `held`'s link for `why`; the node hears a change, not a state. +fn end_link(held: &mut Held, why: Ended, node: &mut Node, now: Instant) { + let Some(session) = held.session.take() else { return }; match why { - Ended::HungUp => say!("netstack: {}'s link ended: it hung up", session.hello().label), + Ended::HungUp => say!("netstack: {}'s link ended: it hung up", held.label), Ended::Fault(fault) => { - say!("netstack: {}'s link ended: it wrote {fault}", session.hello().label); - *faulted = true; + say!("netstack: {}'s link ended: it wrote {fault}", held.label); + held.faulted = true; } } if session.is_up() { - node.link(now, false, draw); + node.link(now, held.iface, false, draw); + } +} + +/// Open, refuse or take back the link each hello asks for. +fn hellos(asked: Vec, links: &mut Vec, node: &mut Node, now: Instant, groups: &[Group]) { + for asked in asked { + let at = links.iter().position(|held| held.mac == asked.mac); + let Some(at) = at else { + if links.len() >= MAX_LINKS as usize { + refuse(&asked, Refusal::Full, &format!("netstack drives at most {MAX_LINKS} links")); + continue; + } + let iface = node.add_link(now, asked.mac, rank(asked.hello.medium), draw); + let label = asked.hello.label.clone(); + let mac = asked.mac; + let session = open(asked, iface, links.len(), groups); + links.push(Held { iface, mac, label, session, faulted: false }); + continue; + }; + let held = &mut links[at]; + if std::mem::take(&mut held.faulted) { + refuse(&asked, Refusal::Faulted, "its last link ended for what it wrote"); + continue; + } + // The kernel closes a dead driver's handles before its end is + // published, so a restarted driver's hello finds the old link already + // hung up. + if let Some(session) = &held.session { + match session.alive() { + Ok(()) => { + refuse(&asked, Refusal::Open, "a link of its address is open"); + continue; + } + Err(why) => end_link(held, why, node, now), + } + } + held.label = asked.hello.label.clone(); + held.session = open(asked, held.iface, at, groups); } } @@ -363,39 +445,11 @@ fn accept_link(links: &Acceptor, pending: &mut Vec) { } } -/// The first link a driver brings, within [`LINK_BOUND`]. -fn first_link(links: &Acceptor, groups: &[Group]) -> Option { - let began = Wall::now(); - let mut pending: Vec = Vec::new(); - let mut poller = watches::Watches::new(1 + MAX_PENDING_LINKS); - loop { - let mut opened = None; - for asked in hear_links(&mut pending) { - match opened { - Some(_) => refuse(&asked, Refusal::OneLink, "netstack drives one link (issues/netstack-drives-one-card.md)"), - None => opened = open(asked, groups), - } - } - if opened.is_some() { - return opened; - } - let left = LINK_BOUND.checked_sub(began.elapsed())?; - poller.watch(links, READABLE, TOKEN_LINKS); - for (i, p) in pending.iter().enumerate() { - p.watch(&mut poller, TOKEN_LINK_PENDING_BASE + i as u64); - } - let mut accept = false; - poller.wait_answers(1, u64::try_from(left.as_nanos()).unwrap_or(u64::MAX), |token, _| accept |= token == TOKEN_LINKS); - if accept { - accept_link(links, &mut pending); - } - } -} - const TOKEN_ACCEPTOR: u64 = 0; const TOKEN_LINKS: u64 = 1; -const TOKEN_LINK: u64 = 2; -const TOKEN_LINK_PENDING_BASE: u64 = 3; +const TOKEN_LINK_PENDING_BASE: u64 = 2; +/// The `n`th link's token. +const TOKEN_LINK_BASE: u64 = TOKEN_LINK_PENDING_BASE + MAX_PENDING_LINKS as u64; const _: () = assert!( toyos_inspect::MAX_SNAPSHOT_BYTES == ipc::MAX_FRAME_LEN as usize, @@ -410,7 +464,7 @@ struct Asked { region: toyos_link::region::Region, } -/// An `inspect` waiting for the driver's counters. +/// An `inspect` waiting for the drivers' counters. struct Inspecting { request: Request, since: Wall, @@ -426,23 +480,16 @@ fn main() { let links = endow::acceptor(toyos_link::PORT) .expect("the manifest declares this program serves `link`"); let groups = [Group::new(toyos_mdns::GROUP_MAC).unwrap_or_else(|| panic!("netstack: mDNS's address is no group's"))]; - let Some(first) = first_link(&links, &groups) else { - say!("netstack: no NIC driver brought a link in {} s; this machine has no network, exiting", LINK_BOUND.as_secs()); - return; - }; - let mac = first.mac(); - let mut link = Some(first); let began = Wall::now(); let clock = || Instant::from_nanos(u64::try_from(began.elapsed().as_nanos()).unwrap_or(u64::MAX)); let config = Config { - mac, receive_buffer: TCP_BUFFER, send_buffer: toyos::net::stream::SEND_BYTES, secrets: secrets(), }; let host = HostName::new(HOSTNAME).unwrap_or_else(|| panic!("netstack: {HOSTNAME:?} is no host name")); - let mut node = Node::new(clock(), config, Some(host), draw) + let mut node = Node::new(clock(), config, Some(host)) .unwrap_or_else(|why| panic!("netstack: the stack refused its configuration: {why:?}")); let name = toyos_mdns::Host::new(HOSTNAME).unwrap_or_else(|_| panic!("netstack: {HOSTNAME:?} is no host name")); @@ -453,7 +500,8 @@ fn main() { let places = places_for(total_mem); node.set_places(places); let mut sockets = serve::Sockets::new(places); - let mut leases = Leases { began, said: None, settled: false }; + let mut leases = Leases { began, said: Vec::new(), settled: false }; + let mut held: Vec = Vec::new(); let mut poller = watches::Watches::new( FIXED_WATCHES + MAX_PLACES + MAX_PENDING_CONNS + serve::LOOKUP_WATCHES, @@ -466,62 +514,73 @@ fn main() { let mut pending: Vec = Vec::new(); let mut pending_links: Vec = Vec::new(); let mut inspecting: Vec = Vec::new(); - // The held link ended for what its driver wrote, and its next hello is - // refused for it. - let mut faulted = false; + // What the drivers have answered of the counters `inspecting` waits for. + let mut gathered: Vec<(IfIndex, Counters)> = Vec::new(); // Accepts the kernel refused since the last it did not. let mut accept_refused: u64 = 0; loop { - // The node begins with its link down, and hears each change of it here. + // The node begins with each link down, and hears each change of it here. let now = clock(); let mut owed = false; - let mut ended = None; - if let Some(session) = &mut link { + let mut ended: Vec<(IfIndex, Ended)> = Vec::new(); + for session in held.iter_mut().filter_map(|h| h.session.as_mut()) { match session.receive(&mut node, now, draw) { - Ok(more) => owed = more, - Err(why) => ended = Some(why), + Ok(more) => owed |= more, + Err(why) => ended.push((session.iface(), why)), } } let now = clock(); if node.next_deadline().is_some_and(|at| at <= now) { node.fire(now, draw); } - if ended.is_none() - && let Some(session) = &mut link - && let Err(why) = session.transmit(&mut node, now, draw) - { - ended = Some(why); - } + let mut open: Vec<&mut Session> = + held.iter_mut().filter_map(|h| h.session.as_mut()).filter(|s| !ended.iter().any(|(at, _)| *at == s.iface())).collect(); + ended.extend(link::transmit(&mut open, &mut node, now, draw).into_iter().map(|(at, fault)| (at, Ended::Fault(fault)))); - // The driver's counters, once it answered them or its time to is up. + // The drivers' counters, once each open link's driver answered them or + // the time to is up. if !inspecting.is_empty() { - // A link opened since the ask is asked in its turn. - if let Some(session) = &mut link { + let mut owing = false; + for session in held.iter_mut().filter_map(|h| h.session.as_mut()) { + if gathered.iter().any(|(at, _)| *at == session.iface()) { + continue; + } + // A link opened since the ask is asked in its turn. session.ask_counters(); + match session.counters() { + Some(c) => gathered.push((session.iface(), c)), + None => owing = true, + } } - let counters = link.as_mut().and_then(Session::counters); let late = inspecting.first().is_some_and(|i| i.since.elapsed() >= INSPECT_BOUND); - if counters.is_some() || late || link.is_none() { + if !owing || late { for waiting in inspecting.drain(..) { - answer_inspect(&waiting.request, link.as_ref().map(|l| (l, counters.as_ref())), &leases, &sockets, &node); + answer_inspect(&waiting.request, &held, &gathered, &leases, &sockets, &node); } + gathered.clear(); } } - if ended.is_none() - && let Some(session) = &mut link - && let Err(why) = session.ring() - { - ended = Some(why); + for session in held.iter_mut().filter_map(|h| h.session.as_mut()) { + if !ended.iter().any(|(at, _)| *at == session.iface()) + && let Err(why) = session.ring() + { + ended.push((session.iface(), why)); + } } - if let Some(why) = ended { - end_link(&mut link, why, &mut node, clock(), &mut faulted); + for (iface, why) in ended { + if let Some(link) = held.iter_mut().find(|h| h.iface == iface) { + end_link(link, why, &mut node, clock()); + } } - sockets.settle(&mut node); + let label = |iface: IfIndex| { + held.iter().position(|h| h.iface == iface).map_or_else(String::new, |n| format!("link {n} ({})", held[n].label)) + }; + sockets.settle(&mut node, label); // Before anything is served: the lease is what gives this machine an // address, a route and its resolvers. - if leases.pass(node.lease()) { + if leases.pass(&node, label) { say!( "netstack: ready, at most {places} places ({} MiB each of {} MiB total)", PLACE_BYTES / (1024 * 1024), @@ -531,8 +590,8 @@ fn main() { poller.watch(&acceptor, READABLE, TOKEN_ACCEPTOR); poller.watch(&links, READABLE, TOKEN_LINKS); - if let Some(session) = &link { - session.watch(&mut poller, TOKEN_LINK); + for (n, session) in held.iter().enumerate().filter_map(|(n, h)| h.session.as_ref().map(|s| (n, s))) { + session.watch(&mut poller, TOKEN_LINK_BASE + n as u64); } for (i, p) in pending_links.iter().enumerate() { p.watch(&mut poller, TOKEN_LINK_PENDING_BASE + i as u64); @@ -570,33 +629,12 @@ fn main() { }); sockets.bridge(&mut node, clock()); - // A driver's connection: a hello with the address the node was made - // with is opened, once a link it replaces has ended. The kernel closes - // a dead driver's handles before its end is published, so a - // restarted driver's hello finds the old link already hung up. + // A driver's connection: each hello is opened on a link of its own, or on + // its address's once that link has ended. if ready.contains(&TOKEN_LINKS) { accept_link(&links, &mut pending_links); } - for asked in hear_links(&mut pending_links) { - if asked.mac != mac { - refuse(&asked, Refusal::OneLink, "its address is not the one netstack drives (issues/netstack-drives-one-card.md)"); - continue; - } - if std::mem::take(&mut faulted) { - refuse(&asked, Refusal::Faulted, "its last link ended for what it wrote"); - continue; - } - if let Some(session) = &link { - match session.alive() { - Ok(()) => { - refuse(&asked, Refusal::OneLink, "a link of its address is open"); - continue; - } - Err(why) => end_link(&mut link, why, &mut node, clock(), &mut faulted), - } - } - link = open(asked, &groups); - } + hellos(hear_links(&mut pending_links), &mut held, &mut node, clock(), &groups); // On a pass that found nothing ready too: otherwise a silent client // is only ever timed out by some other client's traffic. let now_wall = Wall::now(); @@ -697,11 +735,13 @@ fn main() { // not this protocol. if request.payload_len != 0 { request.client.error(toyos::net::ERR_INVALID_INPUT); - } else if let Some(session) = link.as_mut().filter(|_| inspecting.len() < MAX_PENDING_CONNS as usize) { - session.ask_counters(); + } else if held.iter().any(|h| h.session.is_some()) && inspecting.len() < MAX_PENDING_CONNS as usize { + for session in held.iter_mut().filter_map(|h| h.session.as_mut()) { + session.ask_counters(); + } inspecting.push(Inspecting { request, since: Wall::now() }); } else { - answer_inspect(&request, link.as_ref().map(|l| (l, None)), &leases, &sockets, &node); + answer_inspect(&request, &held, &[], &leases, &sockets, &node); } continue; } diff --git a/system/netstack/src/serve.rs b/system/netstack/src/serve.rs index fed88b0a697..d237a5119a6 100644 --- a/system/netstack/src/serve.rs +++ b/system/netstack/src/serve.rs @@ -281,6 +281,8 @@ fn connect_failed(failure: Failure) -> u32 { Failure::Refused => ERR_CONNECTION_REFUSED, Failure::Reset => ERR_CONNECTION_RESET, Failure::TimedOut => ERR_TIMED_OUT, + // The address it was made from went under it: as no address at all. + Failure::AddressGone => ERR_NOT_CONNECTED, Failure::Unreachable(_) | Failure::Prohibited => { say!("netstack: a connect ended {failure:?}"); ERR_OTHER @@ -513,8 +515,8 @@ impl Sockets { /// Everything the node has to say since the last call, to the log and to /// the clients that waited for it, and the table entries of what the node - /// let go. - pub fn settle(&mut self, node: &mut Node) { + /// let go. `label` names a link in a line. + pub fn settle(&mut self, node: &mut Node, label: impl Fn(toyos_net_node::IfIndex) -> String) { let events: Vec = node.drain_stream_events().collect(); for event in events { let (id, answer) = match event { @@ -549,13 +551,16 @@ impl Sockets { match event { Event::Stack { refusal, suppressed: 0 } => say!("netstack: refused {refusal:?}"), Event::Stack { refusal, suppressed } => say!("netstack: refused {refusal:?}, and {suppressed} more by its rule"), - Event::Dhcp { refusal, suppressed: 0 } => say!("netstack: DHCP: refused {refusal:?}"), - Event::Dhcp { refusal, suppressed } => say!("netstack: DHCP: refused {refusal:?}, and {suppressed} more by its rule"), - Event::Name(toyos_mdns::Event::Claimed) => { - say!("netstack: mDNS: no host answered for {HOSTNAME}.local; this machine answers as it") + Event::Dhcp { link, refusal, suppressed: 0 } => say!("netstack: DHCP on {}: refused {refusal:?}", label(link)), + Event::Dhcp { link, refusal, suppressed } => { + say!("netstack: DHCP on {}: refused {refusal:?}, and {suppressed} more by its rule", label(link)) } - Event::Name(toyos_mdns::Event::Lost) => say!( - "netstack: mDNS: another host answered for {HOSTNAME}.local; this machine answers to no name and asks for {HOSTNAME}.local again every {} s", + Event::Name { link, event: toyos_mdns::Event::Claimed } => { + say!("netstack: mDNS on {}: no host answered for {HOSTNAME}.local; this machine answers as it there", label(link)) + } + Event::Name { link, event: toyos_mdns::Event::Lost } => say!( + "netstack: mDNS on {}: another host answered for {HOSTNAME}.local; this machine answers to no name there and asks for {HOSTNAME}.local again every {} s", + label(link), toyos_mdns::RETRY_MS / 1000 ), } diff --git a/system/netstack/udp/src/lib.rs b/system/netstack/udp/src/lib.rs index b948ef8619d..11dc4ec3bd1 100644 --- a/system/netstack/udp/src/lib.rs +++ b/system/netstack/udp/src/lib.rs @@ -15,8 +15,9 @@ //! //! **A datagram waits for its next hop where it was accepted.** One the caller answers //! [`Offer::Waits`] stays in its sender's queue, under that queue's bound, and is passed over -//! until [`Udp::wake`] names the hop: the sender's datagrams to other hops leave meanwhile, and -//! those to one hop leave in the order they were accepted. One whose hop [`Udp::fail`] names is +//! until [`Udp::wake`] names the hop on its interface: the sender's datagrams to other hops leave +//! meanwhile, and those to one hop leave in the order they were accepted. One whose hop on its +//! interface [`Udp::fail`] names is //! dropped, counted, and reported to a connected socket; nothing is kept for another try. //! //! **Refusals are values.** Every refusal is a named [`Counter`] and the [`Error`] the call @@ -42,7 +43,7 @@ use alloc::collections::{BTreeMap, VecDeque}; use alloc::vec::Vec; use core::net::Ipv4Addr; -use toyos_net_ip::{Arrival, Cast, ErrorClass, Flow, Ip, Source, Transport, TransportError, UdpOut}; +use toyos_net_ip::{Arrival, Cast, ErrorClass, Flow, IfIndex, Ip, Source, Transport, TransportError, UdpOut}; use toyos_net_wire::addr::is_martian; use toyos_net_wire::ethernet::MacClass; use toyos_net_wire::ipv4::Ttl; @@ -158,6 +159,8 @@ pub struct Received { pub destination: Ipv4Addr, pub ttl: u8, pub link_broadcast: bool, + /// The interface it arrived by. + pub iface: IfIndex, pub at: Instant, } @@ -189,9 +192,18 @@ struct Stored { destination: Ipv4Addr, ttl: u8, link_broadcast: bool, + iface: IfIndex, at: Instant, } +/// Where the acquisition socket's datagram is from: the source it names, and the interface whose +/// address it acquires, which alone it leaves by. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Origin { + pub iface: IfIndex, + pub source: Ipv4Addr, +} + /// A datagram accepted for sending, with the addressing it was accepted with. #[derive(Debug)] struct Queued { @@ -205,8 +217,11 @@ struct Queued { /// leaves is decided by this, not by what the socket holds then. broadcast: bool, payload: Vec, - /// The next hop whose link address it waits for: not offered until [`Udp::wake`] names it. - waits: Option, + /// The interface the send named, which alone it may leave by. + iface: Option, + /// The next hop, and its interface, whose link address it waits for, or the hop of a link + /// with no room: not offered until [`Udp::wake`] names both. + waits: Option<(IfIndex, Ipv4Addr)>, } impl Queued { @@ -216,6 +231,7 @@ impl Queued { destination: self.destination, ttl: self.ttl, broadcast: self.broadcast, + iface: self.iface, datagram: UdpBuilder { source: self.source_port, destination: self.port, data: &self.payload }, } } @@ -263,9 +279,10 @@ pub enum Sender { pub enum Offer { /// It left [udp]: in a frame, or refused by [ip], which counted it and told its flow. Taken, - /// The link address of this next hop is being asked for: the datagram stays with its sender - /// until [`Udp::wake`] or [`Udp::fail`] names the hop. - Waits(Ipv4Addr), + /// The link address of this next hop on this interface is being asked for, or the interface + /// has no room: the datagram stays with its sender until [`Udp::wake`] or [`Udp::fail`] names + /// the hop on its interface. + Waits { iface: IfIndex, hop: Ipv4Addr }, } /// What [`Udp::serve`] handed the caller. @@ -503,14 +520,14 @@ impl Udp { /// Queues `payload` for `destination:port`. Accepted means queued, not sent. pub fn send_to(&mut self, ip: &mut Ip, id: SocketId, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), Error> { - self.submit(ip, id, None, destination, port, payload) + self.submit(ip, id, (None, None), destination, port, payload) } /// Queues `payload` for the connected peer. pub fn send(&mut self, ip: &mut Ip, id: SocketId, payload: &[u8]) -> Result<(), Error> { let socket = self.socket(id)?; match socket.binding { - Binding::Connected { peer, peer_port, .. } => self.submit(ip, id, None, peer, peer_port.get(), payload), + Binding::Connected { peer, peer_port, .. } => self.submit(ip, id, (None, None), peer, peer_port.get(), payload), Binding::Any | Binding::Specific(_) => { self.count(Counter::NotConnected); Err(Error::Refused(Counter::NotConnected)) @@ -520,11 +537,19 @@ impl Udp { /// The acquisition socket's send, naming its source: 0.0.0.0 to the limited broadcast, or an /// assigned address. - pub fn send_from(&mut self, ip: &mut Ip, id: SocketId, source: Ipv4Addr, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), Error> { - self.submit(ip, id, Some(source), destination, port, payload) + pub fn send_from(&mut self, ip: &mut Ip, id: SocketId, origin: Origin, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), Error> { + self.submit(ip, id, (Some(origin.source), Some(origin.iface)), destination, port, payload) + } + + /// A datagram that leaves by `iface` alone, from the address the socket's binding and that + /// interface give it: one host's datagram to a group on each of its links. + pub fn send_via(&mut self, ip: &mut Ip, id: SocketId, iface: IfIndex, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), Error> { + self.submit(ip, id, (None, Some(iface)), destination, port, payload) } - fn submit(&mut self, ip: &mut Ip, id: SocketId, named: Option, destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), Error> { + /// `from` is the source a send names and the interface it names, either absent. + fn submit(&mut self, ip: &mut Ip, id: SocketId, from: (Option, Option), destination: Ipv4Addr, port: u16, payload: &[u8]) -> Result<(), Error> { + let (named, iface) = from; let socket = self.socket(id)?; if let Some(error) = socket.pending.take() { return Err(Error::Failed(error)); @@ -572,7 +597,7 @@ impl Udp { } (None, Binding::Any) => (Ipv4Addr::UNSPECIFIED, Source::Any), }; - let source = match ip.route(destination, lookup, None) { + let source = match ip.route(destination, lookup, iface) { Ok(route) if lookup == Source::Any => route.source, Ok(_) => source, Err(refusal) => return self.refuse(route_refusal(refusal), me, peer), @@ -582,7 +607,7 @@ impl Udp { return self.refuse(Counter::TxQueueFull, me, peer); } let ttl = if destination.is_multicast() { multicast_ttl } else { ttl }; - socket.tx.push_back(Queued { source, source_port: local_port, destination, port, ttl, broadcast: broadcast_permitted, payload: payload.to_vec(), waits: None }); + socket.tx.push_back(Queued { source, source_port: local_port, destination, port, ttl, broadcast: broadcast_permitted, payload: payload.to_vec(), iface, waits: None }); if !core::mem::replace(&mut socket.offered, true) { self.eligible.push(Sender::Socket(id)); } @@ -611,6 +636,7 @@ impl Udp { destination: d.destination, ttl: d.ttl, link_broadcast: d.link_broadcast, + iface: d.iface, at: d.at, })) } @@ -689,6 +715,7 @@ impl Udp { destination, ttl: arrival.packet.ttl(), link_broadcast: arrival.link == MacClass::Broadcast, + iface: arrival.iface, at: now, }); self.count(Counter::RxDelivered); @@ -770,7 +797,7 @@ impl Udp { taken = Some(at); break; } - Offer::Waits(next_hop) => queued.waits = Some(next_hop), + Offer::Waits { iface, hop } => queued.waits = Some((iface, hop)), } } let taken = taken.and_then(|at| queue.remove(at)).is_some(); @@ -784,14 +811,15 @@ impl Udp { } } - /// [ip] gave `next_hop` up: every datagram that waited for it is dropped and counted, and - /// the connected socket it left from is told. - pub fn fail(&mut self, next_hop: Ipv4Addr) { + /// [ip] gave `next_hop` on `iface` up: every datagram that waited for it there is dropped + /// and counted, and the connected socket it left from is told. One waiting on the same + /// address on another link stays. + pub fn fail(&mut self, iface: IfIndex, next_hop: Ipv4Addr) { let mut lost = Vec::new(); let queues = self.slots.iter_mut().filter_map(|slot| slot.socket.as_mut()).map(|socket| &mut socket.tx).chain([&mut self.closed]); for queue in queues { queue.retain(|queued| { - let waited = queued.waits == Some(next_hop); + let waited = queued.waits == Some((iface, next_hop)); if waited { lost.push(queued.flow()); } @@ -804,9 +832,10 @@ impl Udp { } } - /// `next_hop`'s link address is known: the datagrams that wait for it are offered again. - pub fn wake(&mut self, next_hop: Ipv4Addr) { - self.rouse(|waits| waits == next_hop); + /// `next_hop`'s link address on `iface` is known, or `iface` has room: the datagrams that + /// wait for it there are offered again. + pub fn wake(&mut self, iface: IfIndex, next_hop: Ipv4Addr) { + self.rouse(|waits| waits == (iface, next_hop)); } /// The routes changed: every waiting datagram's next hop is asked for again. @@ -815,7 +844,7 @@ impl Udp { } /// Ends the wait of every datagram whose next hop `woken` names, and offers its sender. - fn rouse(&mut self, woken: impl Fn(Ipv4Addr) -> bool) { + fn rouse(&mut self, woken: impl Fn((IfIndex, Ipv4Addr)) -> bool) { let Self { slots, closed, closed_offered, eligible, .. } = self; let mut rouse = |queue: &mut VecDeque, offered: &mut bool, sender: Sender| { let mut any = false; diff --git a/system/netstack/udp/tests/common/mod.rs b/system/netstack/udp/tests/common/mod.rs index 180e24b78cb..e4e33044332 100644 --- a/system/netstack/udp/tests/common/mod.rs +++ b/system/netstack/udp/tests/common/mod.rs @@ -189,7 +189,7 @@ impl U { /// A's interface up with MAC A and nothing configured. pub fn bare() -> Self { let mut ip = Ip::new(Instant::from_nanos(0), [0x5a; 16]); - let if0 = ip.add_interface(Instant::from_nanos(0), IndividualMac::new(MAC_A).unwrap()); + let if0 = ip.add_interface(Instant::from_nanos(0), IndividualMac::new(MAC_A).unwrap(), toyos_net_ip::Rank(0)); ip.link_up(Instant::from_nanos(0), if0).unwrap(); ip.join(Instant::from_nanos(0), if0, MulticastAddr::new(MDNS).unwrap()).unwrap(); Self::setup_until(&mut ip, Self::instant(0)); @@ -203,7 +203,7 @@ impl U { pub fn uf() -> Self { let mut ip = Ip::new(Instant::from_nanos(0), [0x5a; 16]); let t = Instant::from_nanos(0); - let if0 = ip.add_interface(t, IndividualMac::new(MAC_A).unwrap()); + let if0 = ip.add_interface(t, IndividualMac::new(MAC_A).unwrap(), toyos_net_ip::Rank(0)); ip.link_up(t, if0).unwrap(); ip.add_address(t, if0, A, 24).unwrap(); Self::setup_until(&mut ip, Instant::from_millis(3_000)); @@ -277,8 +277,8 @@ impl U { for event in &events { match event { Event::Unreachable(flow) => self.udp.unreachable(flow), - Event::Resolved { next_hop, .. } => self.udp.wake(*next_hop), - Event::Failed { next_hop, .. } => self.udp.fail(*next_hop), + Event::Resolved { iface, next_hop } => self.udp.wake(*iface, *next_hop), + Event::Failed { iface, next_hop } => self.udp.fail(*iface, *next_hop), _ => {} } } @@ -308,13 +308,14 @@ impl U { let mut framed = false; let served = udp.serve(sender, |out| { let mut buf = [0u8; FRAME]; - match ip.send_udp(now, out, &mut buf) { - Ok(Sent::Frame(n)) => { + match ip.send_udp(now, &usize::MAX, out, &mut buf) { + Ok(Sent::Frame { len: n, .. }) => { frames.push(buf[..n].to_vec()); framed = true; Offer::Taken } - Ok(Sent::Pending(next_hop)) => Offer::Waits(next_hop), + Ok(Sent::Pending { iface, next_hop }) => Offer::Waits { iface, hop: next_hop }, + Ok(Sent::NoRoom { .. }) => unreachable!("every interface has room"), Err(_) => Offer::Taken, } }); diff --git a/system/netstack/udp/tests/send.rs b/system/netstack/udp/tests/send.rs index 72b7c702271..b5554244af6 100644 --- a/system/netstack/udp/tests/send.rs +++ b/system/netstack/udp/tests/send.rs @@ -458,19 +458,19 @@ fn acquisition() -> (U, toyos_net_udp::SocketId) { #[test] fn s_udp_us_028_only_the_acquisition_socket_sends_from_nowhere() { let (mut u, id) = acquisition(); - u.udp.send_from(&mut u.ip, id, ANY, LIMITED, 67, b"discover").unwrap(); + u.udp.send_from(&mut u.ip, id, toyos_net_udp::Origin { iface: u.if0, source: ANY }, LIMITED, 67, b"discover").unwrap(); let out = u.out(); assert_eq!(ip_of(&out[0]).unwrap().source(), ANY); - assert_eq!(u.udp.send_from(&mut u.ip, id, ANY, R, 67, b"x"), refused(Counter::SourceNotPermitted)); + assert_eq!(u.udp.send_from(&mut u.ip, id, toyos_net_udp::Origin { iface: u.if0, source: ANY }, R, 67, b"x"), refused(Counter::SourceNotPermitted)); let other = u.bind(ANY, 5_000).unwrap(); u.udp.set_broadcast(other, true).unwrap(); - assert_eq!(u.udp.send_from(&mut u.ip, other, ANY, LIMITED, 67, b"x"), refused(Counter::SourceNotPermitted)); + assert_eq!(u.udp.send_from(&mut u.ip, other, toyos_net_udp::Origin { iface: u.if0, source: ANY }, LIMITED, 67, b"x"), refused(Counter::SourceNotPermitted)); } #[test] fn s_udp_us_029_the_discover_byte_for_byte() { let (mut u, id) = acquisition(); let discover = hex(V_DHCP_DISCOVER); - u.udp.send_from(&mut u.ip, id, ANY, LIMITED, 67, &discover[42..]).unwrap(); + u.udp.send_from(&mut u.ip, id, toyos_net_udp::Origin { iface: u.if0, source: ANY }, LIMITED, 67, &discover[42..]).unwrap(); assert_eq!(u.out(), [discover]); } diff --git a/system/sshserver/src/main.rs b/system/sshserver/src/main.rs index 926657ca567..dbeecbd0048 100644 --- a/system/sshserver/src/main.rs +++ b/system/sshserver/src/main.rs @@ -153,8 +153,9 @@ fn main() { let Some(mut listener) = listen() else { return }; // Identity and trust are settled after the bind, so that a machine with no - // NIC still reports the network as the reason it is leaving rather than - // minting a key it will never present. + // netstack reports the network as the reason it is leaving rather than + // minting a key it will never present. A machine with no NIC has a + // netstack, which binds the port: it serves once a link brings one. let host = match host_key() { Ok(key) => Arc::new(key), Err(why) => return println!("sshserver: {why}, exiting"), diff --git a/system/virtio-net/src/nic.rs b/system/virtio-net/src/nic.rs index ea49a439964..8e42c2cb53c 100644 --- a/system/virtio-net/src/nic.rs +++ b/system/virtio-net/src/nic.rs @@ -34,6 +34,13 @@ use toyos_virtio::queue::{avail_bytes, desc_bytes, Buffer, Parts, Published, Use /// §5.1.3: the device has a MAC address of its own to read. const VIRTIO_NET_F_MAC: u64 = 1 << 5; +/// §5.1.3: the device reports its link in `status`, and a change of it is a +/// configuration change (§5.1.4.2). +const VIRTIO_NET_F_STATUS: u64 = 1 << 16; +/// §5.1.4: `status`, after the six bytes of `mac`; its bit 0 is +/// `VIRTIO_NET_S_LINK_UP`. +const STATUS_AT: usize = 6; +const VIRTIO_NET_S_LINK_UP: u16 = 1; /// The one MSI-X table entry the kernel programs. const MSIX_ENTRY: u16 = 0; @@ -101,6 +108,8 @@ pub struct VirtioNet { rx: Virtqueue, tx: TxQueue, mac: [u8; 6], + /// The device accepted `VIRTIO_NET_F_STATUS`, and reports its link. + status: bool, } impl VirtioNet { @@ -108,7 +117,8 @@ impl VirtioNet { /// feature negotiation, both queues and their vectors in the order /// `toyos-virtio`'s types fix, which is virtio 1.2 §3.1.1's. pub fn open(dev: PciDev) -> Result { - let Negotiated { setup, mapping, features } = negotiate(&dev, VIRTIO_NET_F_MAC)?; + let Negotiated { setup, mapping, features } = negotiate(&dev, VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS)?; + let status = setup.features() & VIRTIO_NET_F_STATUS != 0; say!("virtio-net: {features}"); let (grant, region) = Grant::alloc(&dev, GRANT_BYTES)?; @@ -137,6 +147,7 @@ impl VirtioNet { rx, tx, mac, + status, }; // Every receive buffer posted before a frame can arrive. @@ -177,6 +188,8 @@ impl VirtioNet { } impl toyos_link::driver::Nic for VirtioNet { + const MEDIUM: toyos_link::wire::Medium = toyos_link::wire::Medium::Wired; + fn mac(&self) -> [u8; 6] { self.mac } @@ -193,12 +206,23 @@ impl toyos_link::driver::Nic for VirtioNet { /// kernel saying this function is no longer this process's, and this /// dies where it can be read. /// - /// The link is unreported: this driver accepts no `VIRTIO_NET_F_STATUS`. + /// The link is `status`'s where the device accepted + /// `VIRTIO_NET_F_STATUS`, read each pass: a change of it raises the + /// configuration vector, the one interrupt, so a pass follows it. A + /// device without the feature reports none. fn begin_pass(&mut self) -> Link { match self.dev.irq() { - Ok(_) | Err(SyscallError::WouldBlock) => Link::Unreported, + Ok(_) | Err(SyscallError::WouldBlock) => {} Err(why) => panic!("virtio-net: this NIC cannot be driven on — {why:?}"), } + if !self.status { + return Link::Unreported; + } + match self.device.device_read16(STATUS_AT) { + Ok(word) if word & VIRTIO_NET_S_LINK_UP != 0 => Link::UpWithoutSpeed, + Ok(_) => Link::Down, + Err(why) => panic!("virtio-net: its link status is not where §5.1.4 puts it — {why}"), + } } /// A buffer is posted again once its frame was taken: one never given diff --git a/tests/common/iommu.rs b/tests/common/iommu.rs index 0546c986e2f..db02b5df36f 100644 --- a/tests/common/iommu.rs +++ b/tests/common/iommu.rs @@ -518,22 +518,22 @@ fn untranslated_nic() -> String { /// A virtio NIC that bypasses the unit ([`Profile::HeadlessNicUntranslated`]) /// is never handed to its driver: the supervisor reads its offer through the -/// claim, refuses it by name, starts no NIC driver, and netstack leaves once -/// its wait for a link is out. Only a QEMU machine can make one: every bench -/// function is behind its unit. +/// claim, refuses it by name and starts no NIC driver, and netstack serves +/// with no link, saying so once its bound on a first lease is out. Only a +/// QEMU machine can make one: every bench function is behind its unit. pub fn a_virtio_function_without_access_platform_is_refused() -> Result<(), String> { let options = BootOptions { profile: Profile::HeadlessNicUntranslated, ..Default::default() }; let mut qemu = QemuInstance::boot_with_options(&netcase(), &[], &[], options); let refused = untranslated_nic(); - let gone = "netstack: no NIC driver brought a link in 30 s; this machine has no network, exiting"; + let gone = "netstack: DHCP: no lease as toyos-t14 on any link in"; let mut text = qemu.boot_log().to_string(); qemu::await_guest(&mut qemu, &mut text, "the supervisor's refusal", |t| t.contains(&refused)).map_err(|e| format!("{e}\n{text}"))?; - // A quiet guest, for netstack's bound on its first link: a ceiling past it. + // A quiet guest, for netstack's bound on its first lease: a ceiling past it. if !text.contains(gone) { text.push_str(&qemu.drain_until(std::time::Duration::from_secs(90), |l| l.contains(gone))); } if !text.contains(gone) { - return Err(format!("netstack did not leave a machine whose NIC has no driver\n{text}")); + return Err(format!("netstack did not say a machine whose NIC has no driver has no address\n{text}")); } let log = Serial::named("boot console", text); log.must_be_clean()?; diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 0e18fd74c0d..07c1ee86a0e 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -701,6 +701,9 @@ pub enum Profile { /// [`Profile::Headless`] with an `e1000e`, QEMU's 82574L, where its virtio /// NIC was: a NIC the image's rows name no machine for. HeadlessE1000e, + /// [`Profile::Headless`] with two virtio NICs, each on a user network of + /// its own: [`TWO_NICS`]. + HeadlessTwoNics, /// [`Profile::Headless`] with the kernel's virtio console created without /// `iommu_platform=on`: a device the kernel itself must tell physical /// addresses. @@ -809,6 +812,7 @@ impl Profile { | Self::HeadlessNicBehindPort | Self::HeadlessNicUntranslated | Self::HeadlessE1000e + | Self::HeadlessTwoNics | Self::HeadlessConsoleUntranslated | Self::HeadlessTwoEdu | Self::Metal @@ -938,8 +942,15 @@ enum Nic { /// QEMU's `e1000e`, which the unit decodes as it does every function /// that is not virtio. E1000e, + /// Two virtio NICs: [`TWO_NICS`]. + VirtioPair, } +/// [`Profile::HeadlessTwoNics`]'s NICs, in the order QEMU creates them: each +/// device's id, which QMP's `set_link` names, and its user network, which +/// QEMU's host is `.2` of. The first is QEMU's default network. +pub const TWO_NICS: [(&str, &str); 2] = [("nic0", "10.0.2.0/24"), ("nic1", "10.0.3.0/24")]; + /// Everything a profile decides about the machine, in one table. A new /// variant answers every question here or does not compile — which `self != /// Profile::Metal` did the opposite of: it handed anything that was not @@ -1142,6 +1153,7 @@ impl Profile { Self::HeadlessNicBehindPort => Shape { nic: Nic::BehindPort, ..Self::Headless.shape() }, Self::HeadlessNicUntranslated => Shape { nic: Nic::Untranslated, ..Self::Headless.shape() }, Self::HeadlessE1000e => Shape { nic: Nic::E1000e, ..Self::Headless.shape() }, + Self::HeadlessTwoNics => Shape { nic: Nic::VirtioPair, ..Self::Headless.shape() }, Self::HeadlessConsoleUntranslated => Shape { console_bypasses: true, ..Self::Headless.shape() }, Self::HeadlessTwoEdu => Shape { probes: Probes::Edus, ..Self::Headless.shape() }, Self::HeadlessUsbSpare => Shape { @@ -2175,6 +2187,21 @@ fn shutdown_reason(bytes: &[u8]) -> Option { Some(value.to_string()) } +/// An open QMP connection for a NIC's link: the carrier its guest driver +/// reads. +pub struct QmpLink(Qmp); + +impl QmpLink { + pub fn open(socket: &Path) -> Self { + Self(Qmp::connect(socket)) + } + + /// The NIC `id`'s link up or down, as its driver will read it. + pub fn set(&mut self, id: &str, up: bool) { + self.0.execute(&format!("{{\"execute\":\"set_link\",\"arguments\":{{\"name\":\"{id}\",\"up\":{up}}}}}")); + } +} + /// An open QMP connection to QEMU's human monitor, for the questions QMP has /// no command of its own for. pub struct QmpMonitor(Qmp); @@ -2514,6 +2541,15 @@ fn qemu_command( Nic::E1000e => { qemu.arg("-netdev").arg(&netdev).arg("-device").arg("e1000e,netdev=net0,romfile="); } + Nic::VirtioPair => { + for (n, (id, net)) in TWO_NICS.iter().enumerate() { + let netdev = if n == 0 { netdev.clone() } else { format!("user,id=net{n},net={net}") }; + qemu.arg("-netdev").arg(&netdev).arg("-device").arg(format!( + "virtio-net-pci-non-transitional,netdev=net{n},id={id},mac=52:54:00:12:34:5{}{platform}", + 6 + n + )); + } + } Nic::BehindPort => { qemu.arg("-device").arg("pcie-root-port,id=nicport,chassis=9"); qemu.arg("-netdev").arg(&netdev).arg("-device").arg(format!( diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index c15d6a9a55c..73a49c9221e 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -475,9 +475,9 @@ pub fn persistent_dir() -> Option<(String, String, String)> { } /// What a transfer row reports of the stack beneath it, as netstack's `inspect` answers it: [tcp]'s -/// and the card's counters. Those a card does not report are left out. +/// counters, and the cards' summed over netstack's links. Those no card reports are left out. pub fn net_counters() -> Result, String> { - const READ: [&str; 33] = [ + const READ: [&str; 25] = [ "net.tcp.sent.bytes", "net.tcp.retransmit.bytes", "net.tcp.rto", @@ -503,23 +503,30 @@ pub fn net_counters() -> Result, String> { "net.tcp.hystart.resumed", "net.tcp.hystart.exits", "net.tcp.hystart.exit.cwnd.bytes", - "net.transmit.full", - "net.transmit.wake_armed", - "net.transmit.wake_taken", - "net.transmit.full.ns", - "net.transmit.unreported", - "net.errors.missed", - "net.wire.sent", - "net.wire.received", + ]; + /// Each link's, under `net.link..`. + const CARD: [&str; 8] = [ + "transmit.full", + "transmit.wake_armed", + "transmit.wake_taken", + "transmit.full.ns", + "transmit.unreported", + "errors.missed", + "wire.sent", + "wire.received", ]; let snap = inspect::ask(toyos_inspect::NET)?; - Ok(READ - .iter() - .filter_map(|key| match snap.get(*key) { - Some(toyos_inspect::Value::U64(v)) => Some((key.to_string(), *v)), - _ => None, - }) - .collect()) + let number = |key: &str| match snap.get(key) { + Some(toyos_inspect::Value::U64(v)) => Some(*v), + _ => None, + }; + let links = number("net.links").unwrap_or(0); + let tcp = READ.iter().filter_map(|key| number(key).map(|v| (key.to_string(), v))); + let cards = CARD.iter().filter_map(|key| { + let each: Vec = (0..links).filter_map(|n| number(&format!("net.link.{n}.{key}"))).collect(); + (!each.is_empty()).then(|| (format!("net.{key}"), each.iter().sum())) + }); + Ok(tcp.chain(cards).collect()) } /// What a row's own connections say of themselves: nothing on ToyOS, whose @@ -538,9 +545,25 @@ impl StreamLog { } } -/// What carries a connection on ToyOS: netstack drives one card, Ethernet. -pub fn link_of(_local: std::net::IpAddr) -> Result<(osbench_lan::link::Link, String), String> { - Ok((osbench_lan::link::Link::Wired, "wired netstack's one Ethernet card".into())) +/// What carries a connection on ToyOS: the netstack link whose lease holds its local address. +pub fn link_of(local: std::net::IpAddr) -> Result<(osbench_lan::link::Link, String), String> { + let snap = inspect::ask(toyos_inspect::NET)?; + let text = |key: String| match snap.get(&key) { + Some(toyos_inspect::Value::Text(t)) => Some(t.clone()), + _ => None, + }; + let links = match snap.get("net.links") { + Some(toyos_inspect::Value::U64(n)) => *n, + _ => 0, + }; + let held = (0..links).find(|n| text(format!("net.link.{n}.lease.address")).is_some_and(|a| a.split('/').next() == Some(&local.to_string()))); + let Some(n) = held else { return Err(format!("no netstack link holds {local}")) }; + let driver = text(format!("net.link.{n}.driver")).unwrap_or_default(); + match text(format!("net.link.{n}.medium")).as_deref() { + Some("wired") => Ok((osbench_lan::link::Link::Wired, format!("netstack's wired link, {driver}"))), + Some("wireless") => Ok((osbench_lan::link::Link::Wifi, format!("netstack's wireless link, {driver}"))), + other => Err(format!("netstack's link {n} is carried over {other:?}")), + } } /// ToyOS reads no temperature: the kernel counts none diff --git a/tests/guest/rust-tests/Cargo.lock b/tests/guest/rust-tests/Cargo.lock index cf129ddd534..2d140087f04 100644 --- a/tests/guest/rust-tests/Cargo.lock +++ b/tests/guest/rust-tests/Cargo.lock @@ -296,6 +296,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inspect" +version = "0.1.0" +dependencies = [ + "toyos", + "toyos-abi", + "toyos-inspect", +] + [[package]] name = "jni" version = "0.22.4" @@ -980,6 +989,7 @@ dependencies = [ "acpiserver-api", "cpal", "getrandom", + "inspect", "libloading", "logkeeper-api", "memmap2", diff --git a/tests/guest/rust-tests/Cargo.toml b/tests/guest/rust-tests/Cargo.toml index b2def872cd4..e9ff628ce37 100644 --- a/tests/guest/rust-tests/Cargo.toml +++ b/tests/guest/rust-tests/Cargo.toml @@ -11,6 +11,8 @@ toyos = { path = "../../../sdk/toyos", features = ["std"] } toyos-window = { path = "../../../sdk/window" } toyos-tco = { path = "../../../lib/tco" } toyos-inspect = { path = "../../../lib/inspect" } +# The reader's question to netstack, for `link_failover`'s per-link counts. +inspect = { path = "../../../system/inspect" } # A login session opened by a job, for `app_view`'s grants. toyos-session = { path = "../../../lib/session" } # The log as logkeeper serves it, and the asking for it, for `counters_metal`'s diff --git a/tests/guest/rust-tests/src/bin/link_failover.rs b/tests/guest/rust-tests/src/bin/link_failover.rs new file mode 100644 index 00000000000..da1b47d7165 --- /dev/null +++ b/tests/guest/rust-tests/src/bin/link_failover.rs @@ -0,0 +1,129 @@ +//! Traffic fails over to the second of two links when the first goes down, +//! and returns when it comes back. +//! +//! On a machine with two virtio NICs, each on a QEMU user network of its own, +//! this job waits for both links' leases, then sends datagrams to an address +//! off both networks: they leave by the link netstack's default route takes, +//! which netstack's own count of each link's frames (`inspect net.link..sent`) +//! names. It asks the harness, on a stream over the other link, to set the +//! first link's NIC down; the next datagrams leave by the other link and none +//! by the first. Asked to set it up again, the datagrams return to it. + +use std::collections::BTreeMap; +use std::io::{BufRead, BufReader, Write}; +use std::net::{Ipv4Addr, SocketAddr, TcpStream, UdpSocket}; +use std::time::{Duration, Instant}; + +use toyos_inspect::Value; + +/// Off both user networks: RFC 5737's TEST-NET-1, which leaves by a default +/// route and answers nothing. +const FAR: (Ipv4Addr, u16) = (Ipv4Addr::new(192, 0, 2, 1), 9); + +/// Datagrams a round sends: more than the frames a link carries of its own +/// in the time a round takes, so a round's link is the one past it. +const ROUND: u64 = 64; + +/// A liveness ceiling on each wait, never a timing. +const BOUND: Duration = Duration::from_secs(90); + +/// netstack's snapshot. +fn snapshot() -> BTreeMap { + inspect::ask(toyos_inspect::NET).unwrap_or_else(|why| panic!("link_failover: netstack's snapshot: {why}")) +} + +fn text<'a>(snap: &'a BTreeMap, key: &str) -> Option<&'a str> { + match snap.get(key) { + Some(Value::Text(text)) => Some(text), + _ => None, + } +} + +/// The frames netstack has handed link `n`. +fn sent(snap: &BTreeMap, n: usize) -> u64 { + match snap.get(&format!("net.link.{n}.sent")) { + Some(Value::U64(sent)) => *sent, + other => panic!("link_failover: link {n}'s sent count is {other:?}"), + } +} + +/// Polls netstack's snapshot until `done` holds of it. +fn until(what: &str, mut done: impl FnMut(&BTreeMap) -> bool) -> BTreeMap { + let by = Instant::now() + BOUND; + loop { + let snap = snapshot(); + if done(&snap) { + return snap; + } + assert!(Instant::now() < by, "link_failover: {what} within {BOUND:?}; netstack says {snap:?}"); + std::thread::yield_now(); + } +} + +/// A round of datagrams, and each link's frames from before it to once one +/// link has carried the round. +fn round(socket: &UdpSocket) -> [u64; 2] { + let before = snapshot(); + let before = [sent(&before, 0), sent(&before, 1)]; + for n in 0..ROUND { + socket.send_to(&n.to_le_bytes(), FAR).expect("link_failover: a datagram queued"); + } + let after = until("a link carrying the round", |snap| (0..2).any(|n| sent(snap, n) >= before[n] + ROUND)); + [sent(&after, 0) - before[0], sent(&after, 1) - before[1]] +} + +/// The link carrying a round, and the other. +fn carried(by: [u64; 2]) -> (usize, usize) { + if by[0] >= ROUND { (0, 1) } else { (1, 0) } +} + +fn main() { + let port: u16 = std::env::args() + .nth(1) + .and_then(|port| port.parse().ok()) + .expect("link_failover: the harness's port is this job's argument"); + let snap = until("both links' leases", |snap| { + snap.get("net.links") == Some(&Value::U64(2)) && (0..2).all(|n| snap.get(&format!("net.link.{n}.lease.held")) == Some(&Value::Bool(true))) + }); + // Each link's network, by its lease's address: QEMU's host is `.2` of it. + let network = |n: usize| { + let address = text(&snap, &format!("net.link.{n}.lease.address")).unwrap_or_else(|| panic!("link_failover: link {n} holds no lease")); + let address: Ipv4Addr = address.split('/').next().and_then(|a| a.parse().ok()).expect("link_failover: an address"); + let [a, b, c, _] = address.octets(); + Ipv4Addr::new(a, b, c, 0) + }; + let networks = [network(0), network(1)]; + + let socket = UdpSocket::bind("0.0.0.0:0").expect("link_failover: a datagram socket"); + let (first, second) = carried(round(&socket)); + println!(" PASS: a round left by link {first}, on {}", networks[first]); + + // The harness, on a stream over the second link: its network's own host. + let [a, b, c, _] = networks[second].octets(); + let mut control = TcpStream::connect(SocketAddr::from((Ipv4Addr::new(a, b, c, 2), port))).expect("link_failover: a stream to the harness over the second link"); + control.set_read_timeout(Some(BOUND)).expect("link_failover: a read timeout"); + let mut answers = BufReader::new(control.try_clone().expect("link_failover: the stream's reader")); + let mut ask = |word: &str| { + writeln!(control, "{word} {}", networks[first]).expect("link_failover: the ask"); + let mut line = String::new(); + answers.read_line(&mut line).expect("link_failover: the harness's answer"); + assert_eq!(line, format!("{word}\n"), "link_failover: the harness answered {line:?}"); + }; + + ask("down"); + until("the first link down", |snap| text(snap, &format!("net.link.{first}.state")) == Some("down")); + let by = round(&socket); + assert_eq!(carried(by).0, second, "link_failover: the round left by {by:?}"); + assert_eq!(by[first], 0, "link_failover: nothing leaves by a link that is down"); + println!(" PASS: with link {first} down, a round left by link {second} and none by link {first}"); + + ask("up"); + until("the first link up", |snap| text(snap, &format!("net.link.{first}.state")) == Some("up")); + // Its lease verified again, the default route is the first link's. + let by = Instant::now() + BOUND; + while carried(round(&socket)).0 != first { + assert!(Instant::now() < by, "link_failover: no round left by link {first} within {BOUND:?} of its return"); + } + println!(" PASS: with link {first} up again, a round left by it"); + println!("link_failover: traffic failed over to the second link and returned to the first"); +} diff --git a/tests/guest/rust-tests/src/bin/netstack_answers.rs b/tests/guest/rust-tests/src/bin/netstack_answers.rs deleted file mode 100644 index aad6f6498d3..00000000000 --- a/tests/guest/rust-tests/src/bin/netstack_answers.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! One request netstack answers: a UDP bind on any address. netstack serves -//! once a NIC driver's link is open, which the driver opens after its card's -//! bring-up, so the answer follows every line that bring-up logged. Exits 0 -//! on the bind, and 1 naming the refusal. - -use std::net::{Ipv4Addr, UdpSocket}; -use std::process::ExitCode; - -fn main() -> ExitCode { - match UdpSocket::bind((Ipv4Addr::UNSPECIFIED, 0)) { - Ok(socket) => { - println!("netstack_answers: bound {:?}", socket.local_addr()); - ExitCode::SUCCESS - } - Err(e) => { - println!("netstack_answers: the bind was refused: {e}"); - ExitCode::FAILURE - } - } -} diff --git a/tests/guest/rust-tests/src/bin/netstack_connect_killed.rs b/tests/guest/rust-tests/src/bin/netstack_connect_killed.rs index 5b756161a36..c504b944a14 100644 --- a/tests/guest/rust-tests/src/bin/netstack_connect_killed.rs +++ b/tests/guest/rust-tests/src/bin/netstack_connect_killed.rs @@ -49,7 +49,10 @@ fn main() { } fn test() { - until(|snapshot| snapshot.get("net.lease.held") == Some(&Value::Bool(true)), "netstack held a lease"); + until( + |snapshot| snapshot.iter().any(|(key, value)| key.starts_with("net.link.") && key.ends_with(".lease.held") && *value == Value::Bool(true)), + "netstack held a lease on a link", + ); let before = held(&snapshot()); let connect = || Command::new(SELF_PATH).arg("connect").stdin(Stdio::null()).spawn().expect("a connecting child"); let (mut killed, mut survivor) = (connect(), connect()); diff --git a/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs b/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs index 997f013eade..5616f971b1e 100644 --- a/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs +++ b/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs @@ -4,10 +4,10 @@ //! `/system/bin/sshserver` has one clean exit for a machine with no network and it is keyed //! on an error kind: `ErrorKind::NotConnected`, which is what //! `NetError::NetstackNotFound` becomes. Anything else panics, by design — a -//! machine that *has* a NIC and cannot bind must be loud. On a NIC-less machine -//! netstack prints its line and exits, and whether sshserver took the quiet arm or put a -//! tokio backtrace across the boot depended on which side of netstack's teardown -//! its bind landed. +//! machine that *has* a NIC and cannot bind must be loud. netstack that exits, as it +//! once did on a NIC-less machine, can do so under a bind, and whether sshserver took +//! the quiet arm or put a tokio backtrace across the boot depended on which side of +//! netstack's teardown its bind landed. //! //! **The race is not staged here. The sequence is.** What made the defect hard //! to see is that it is a handful of instructions wide in a real boot; what diff --git a/tests/images/benchcase.toml b/tests/images/benchcase.toml index 7e4acdfa46c..40fbc32f588 100644 --- a/tests/images/benchcase.toml +++ b/tests/images/benchcase.toml @@ -36,7 +36,7 @@ receives = ["power"] # netstack over the T14's wired I219, the only card a bench has that ToyOS # drives: `osbench lan` reaches its peer and `osbench internet` the internet # through it. On the AMD bench, which has none, the supervisor starts no -# driver, netstack leaves, and both say so. +# driver, netstack has no address, and both say so. [programs.netstack] service = true serves = ["netstack", "link"] diff --git a/tests/images/metalcase.toml b/tests/images/metalcase.toml index 0e7dbaa0941..073aee6b7b6 100644 --- a/tests/images/metalcase.toml +++ b/tests/images/metalcase.toml @@ -45,8 +45,8 @@ drives = ["pci:04:03:*"] receives = ["sound-output"] syscap = ["rt"] -# netstack over the link a NIC driver brings; on a machine with no virtio NIC -# the supervisor starts no driver, and netstack leaves. +# netstack over every link a NIC driver brings; on a machine with no virtio NIC +# the supervisor starts no driver, and netstack serves with no address. [programs.netstack] service = true serves = ["netstack", "link"] diff --git a/tests/images/netcase.toml b/tests/images/netcase.toml index 19598e0ecec..d0684da33a5 100644 --- a/tests/images/netcase.toml +++ b/tests/images/netcase.toml @@ -2,9 +2,9 @@ # NVMe controller and soundserver's virtio-sound beside it: `iommu_virtio_platform`'s # machine, where every virtio function is a process's claim or a kernel driver's. # -# netstack serves once a NIC driver says hello, and leaves a machine none says -# hello on within 30 s. Every NIC driver is here, so whichever NIC the machine -# has comes up with no row naming it. +# netstack serves every link a NIC driver brings, from the start. Every NIC +# driver is here, so whichever NICs the machine has come up with no row +# naming them. # # test-runner is here for the same reason it is in metalcase: it makes the boot # announce itself. diff --git a/tests/toyos.rs b/tests/toyos.rs index 4dc4052e248..6e76b1fdeff 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -139,12 +139,9 @@ const RUST_SKIP: &[&str] = &[ // It measures the T14's I219, which no guest has: the // `i219_messages_on_its_msi_vector` metal row runs it. "i219_msi", - // It waits for netstack to answer, which follows the I219's bring-up: the - // `i219_leaves_ulp_unarmed` metal row runs it on tests/images/benchcase.toml. - "netstack_answers", - // It waits for a lease from the bench's router over the T14's I219: the - // `i219_carries_netstacks_link` metal row runs it on - // tests/images/benchcase.toml. + // It waits for a lease from the bench's router over the T14's I219, which + // follows the card's bring-up: the `i219_carries_netstacks_link` and + // `i219_leaves_ulp_unarmed` metal rows run it on tests/images/benchcase.toml. "lan_lease", // Its product is the T14's counters across an idle span and a spin on // every CPU, which the `counters` metal row judges; on a guest it would be @@ -269,6 +266,9 @@ const RUST_SKIP: &[&str] = &[ // netstack, which a shared boot runs none of: // `a_client_killed_mid_connect_gives_its_place_back` runs it. "netstack_connect_killed", + // It needs two NICs and a harness that sets one's link down: + // `traffic_leaves_by_the_second_link_while_the_first_is_down` runs it. + "link_failover", // It needs a launcher: `a_launch_gives_each_unsent_stdio_slot_a_null` // runs it on tests/images/launchcase.toml. "launch_unsent_stdio", @@ -676,6 +676,18 @@ const MACHINE_TESTS: &[&str] = &[ // netstack takes to give the address up, or the test is red. x86-64 alone // (`qemu::SUITE_ARCH`). "a_client_killed_mid_connect_gives_its_place_back", + // netstack over two links: the default route leaves by the first, by the + // second while QMP holds the first's NIC down, and by the first again + // when it is up. Two drivers' links, a NIC's carrier and netstack's + // routes cross three processes and two devices, which the node's and the + // shard's host tests of rank, failover and return hold below the drivers. + // QEMU x86-64 (`qemu::SUITE_ARCH`), on two user networks; the datagrams go + // to RFC 5737's TEST-NET-1, which answers nothing, and the harness is the + // peer on the host's loopback, so it depends on no public network. The networks are not + // `restrict=on`: with it the job's first datagram off both networks was + // refused for want of a route, and the default route is what is tested. + // Order and counts only. + "traffic_leaves_by_the_second_link_while_the_first_is_down", // Login sessions: their owner parents, signs out and ends with each, and // the machine's reboot waits for them until forced, which only processes // in one boot show, as order, completion and counts. The reboot ends the @@ -1220,7 +1232,7 @@ const METAL: &[(&str, metal::Metal)] = &[ // full reset, which the stub only assumes. The T14 only: the AMD bench // has no card ToyOS drives. It rides `osbench`'s boot, whose i219 row // drives the card, and reads two of the driver's lines once its job - // has had netstack answer, which it does only after the bring-up. + // has seen the lease, which follows the bring-up. "i219_leaves_ulp_unarmed", metal::Metal { arms: BENCHCASE_ALONE, judge: |b| i219_leaves_ulp_unarmed(b[0]) }, ), @@ -1811,18 +1823,17 @@ const OSBENCH_INTERNET: &[metal::Arm] = &[metal::Arm { ..metal::once("netbenchcase", "tests/images/benchcase.toml", &[], &["osbench_internet"]) }]; -/// [`BENCHCASE`]'s boot, with two jobs: netstack answering, which it does only -/// once the card's bring-up has logged, and its lease landing. Without them -/// the list's `reboot` lands while the driver is still reaching the PHY. -const BENCHCASE_ALONE: &[metal::Arm] = - &[metal::once("benchcase", "tests/images/benchcase.toml", &[], &["test_rs_netstack_answers", "test_rs_lan_lease"])]; +/// [`BENCHCASE`]'s boot, with one job: the lease landing, which follows the +/// card's bring-up and every line it logged. Without it the list's `reboot` +/// lands while the driver is still reaching the PHY. +const BENCHCASE_ALONE: &[metal::Arm] = &[metal::once("benchcase", "tests/images/benchcase.toml", &[], &["test_rs_lan_lease"])]; /// The I219 driver's account of its part's start: what its wake found of /// Ultra Low Power and disarmed, and the configuration its bring-up read back /// unarmed. Both lines are printed, because what was found is what the next /// boot's diagnosis reads. fn i219_leaves_ulp_unarmed(boot: &metal::Readback) -> Result<(), String> { - boot.job_passed("test_rs_netstack_answers")?; + boot.job_passed("test_rs_lan_lease")?; let log = boot.log(); let woke = log.must_say("i219: before the reset the PHY was asked:")?; let left = log.must_say("it is left with ULP configuration")?; @@ -4703,6 +4714,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "udp_receive_times_out" => udp_receive_times_out(), "a_link_ends_under_a_held_stream_and_comes_back" => a_link_ends_under_a_held_stream_and_comes_back(), "a_client_killed_mid_connect_gives_its_place_back" => a_client_killed_mid_connect_gives_its_place_back(), + "traffic_leaves_by_the_second_link_while_the_first_is_down" => traffic_leaves_by_the_second_link_while_the_first_is_down(), "session_lifecycle" => common::sessionlifecycle::session_lifecycle(), "a_launch_gives_each_unsent_stdio_slot_a_null" => common::launchstdio::a_launch_gives_each_unsent_stdio_slot_a_null(), "a_row_reaches_its_own_state_and_no_other" => common::stateview::a_row_reaches_its_own_state_and_no_other(), @@ -5792,6 +5804,57 @@ fn peer_verdict(heard: std::thread::JoinHandle>) -> String } } +/// `test_rs_link_failover` on two virtio NICs: the harness, asked over the +/// second link's network, sets the first link's NIC down and then up. +fn traffic_leaves_by_the_second_link_while_the_first_is_down() -> Result<(), String> { + use std::io::{BufRead, BufReader, Write}; + const JOB: &str = "link_failover"; + /// A liveness ceiling on the job and the peer's waits, never a timing. + const CEILING: Duration = Duration::from_secs(300); + let tests = compile::repo_root().join("tests/guest/rust-tests"); + let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &tests, JOB); + let config = compile::repo_root().join("tests/images/netcase.toml"); + let options = BootOptions { profile: qemu::Profile::HeadlessTwoNics, qmp: true, ..BootOptions::default() }; + let mut qemu = QemuInstance::boot_with_options(&config, &[], &[(JOB.to_string(), bin)], options); + serial::Serial::boot(&qemu).must_be_clean()?; + let peer = std::net::TcpListener::bind("127.0.0.1:0").map_err(|e| format!("no loopback listener: {e}"))?; + let port = peer.local_addr().map_err(|e| format!("the listener's port: {e}"))?.port(); + let socket = qemu.qmp_socket().to_path_buf(); + let heard = std::thread::spawn(move || -> Result<(), String> { + let (conn, _) = peer.accept().map_err(|e| format!("the guest's connect: {e}"))?; + conn.set_read_timeout(Some(CEILING)).map_err(|e| e.to_string())?; + let mut reader = BufReader::new(conn.try_clone().map_err(|e| e.to_string())?); + let mut answer = conn; + let mut link = qemu::QmpLink::open(&socket); + for word in ["down", "up"] { + let mut line = String::new(); + reader.read_line(&mut line).map_err(|e| format!("the job's {word:?} ask: {e}"))?; + let Some((said, network)) = line.trim_end().split_once(' ') else { return Err(format!("the job asked {line:?}")) }; + if said != word { + return Err(format!("the job asked {line:?} where {word:?} was owed")); + } + let Some((id, _)) = qemu::TWO_NICS.iter().find(|(_, net)| net.split('/').next() == Some(network)) else { + return Err(format!("the job named {network:?}, which is neither NIC's network")); + }; + link.set(id, word == "up"); + answer.write_all(format!("{word}\n").as_bytes()).map_err(|e| format!("the answer to {word:?}: {e}"))?; + } + Ok(()) + }); + let result = qemu.run_test(&format!("test_rs_{JOB} {port}"), CEILING); + if let Some(why) = &result.error { + return Err(format!("{why}\n{}\nthe job said:\n{}", peer_verdict(heard), result.stdout)); + } + if result.exit_code != Some(0) { + return Err(format!("the job ended {:?}:\n{}", result.exit_code, result.stdout)); + } + heard.join().map_err(|_| "the peer's thread panicked".to_string())??; + for line in result.stdout.lines().filter(|l| l.contains("PASS: ")) { + eprintln!(" [links] {}", line.trim()); + } + Ok(()) +} + /// `test_rs_link_restart` on `tests/images/linkcase.toml`'s boot: the peer /// answers each line the job writes on one connection, the one before the /// job ends its driver and the one after it starts another, then reads the