diff --git a/.build/build-binaries.sh b/.build/build-binaries.sh new file mode 100755 index 0000000..8e7f420 --- /dev/null +++ b/.build/build-binaries.sh @@ -0,0 +1,329 @@ +#!/bin/bash +# +# build-binaries.sh builds the component binaries from the pinned upstream +# sources and stages them under ./binaries/ so they can be shipped in the +# mlab-node Debian package. The version pins are the same tags the historical +# docker-compose deployment ran (via the corresponding container images), but +# everything is built from source here: no container images, no Docker, no +# skopeo — just git, go and a C toolchain (for scamper and the cgo builds). +# +# Building from source (rather than extracting binaries from the images, as an +# earlier iteration of this script did) is what makes non-amd64 packages +# possible at all: M-Lab publishes amd64-only images, but the sources build +# anywhere Go and gcc do. +# +# Each Go component replicates its upstream Dockerfile's build: same package +# path, same CGO setting, same -ldflags (version stamps). Divergences from +# upstream: +# - ndt-server is built CGO+glibc-dynamic instead of CGO+musl-static on +# Alpine; it runs natively on Debian (dh_shlibdeps picks up the libc dep). +# - all binaries are linked with -s -w (no symbol table, no DWARF), roughly +# halving them. Panic backtraces stay fully symbolized (Go's pclntab is +# unaffected); only delve/pprof against the shipped binary lose fidelity. +# +# Caching: every component carries its own stamp under binaries/.stamps/, +# keyed on its recipe (pinned tag, package path, build flags, toolchain +# versions). Re-running the script rebuilds only components whose recipe +# changed — bumping one version pin rebuilds that one component. Delete +# binaries/ (or bump STAMP_SCHEMA below) to force a full rebuild. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +OUT_DIR="${REPO_DIR}/binaries" +STAMP_DIR="${OUT_DIR}/.stamps" + +# Salt mixed into every recipe key. Bump it to invalidate all cached +# components at once, e.g. when the build logic itself changes in a way the +# per-component keys cannot see. +STAMP_SCHEMA=1 + +# Version pins. These mirror the image tags pinned in the original +# docker-compose.yml (each image was built from the repo tag of the same name). +# NOTE: the annotation2 schema generator intentionally comes from an older +# uuid-annotator tag (v0.5.8) than the running annotator (v0.5.10), matching +# the compose configuration. +# NOTE: heartbeat lives in the m-lab/locate repository (cmd/heartbeat); the +# measurementlab/heartbeat image was built from locate's tags. +NDT_SERVER_VERSION="v0.25.3" +LOCATE_VERSION="v0.19.1" +UUID_ANNOTATOR_VERSION="v0.5.10" +UUID_ANNOTATOR_SCHEMA_VERSION="v0.5.8" +JOSTLER_VERSION="v1.1.4" +TRACEROUTE_VERSION="v0.12.0" +AUTOJOIN_VERSION="v0.2.13" +NODE_EXPORTER_VERSION="v1.9.0" + +# scamper is a C tool; the traceroute-caller repo vendors the exact tarball its +# image builds (third_party/scamper), so we build that same tarball. +SCAMPER_DIST="scamper-cvs-20230302" + +# ndt-server's go.mod carries a bare "go 1.25" directive, which a system go +# older than 1.25 mis-resolves to a non-existent "go1.25" toolchain download. +# Pin a real release; go fetches it on demand if the system go is older. +NDT_GO_TOOLCHAIN="go1.25.11" + +for tool in go git gcc make file; do + command -v "${tool}" >/dev/null 2>&1 || { + echo "ERROR: required build tool '${tool}' not found in PATH" >&2 + exit 1 + } +done + +# Toolchain fingerprints for the recipe keys: a Go upgrade invalidates every +# component; a gcc/glibc change only the native (cgo/C) ones, whose output — +# including the computed libc6 dependency — depends on the host toolchain. +GO_VERSION="$(go version | awk '{print $3}')" +NATIVE_TOOLCHAIN="gcc=$(gcc -dumpfullversion) libc=$(getconf GNU_LIBC_VERSION)" + +# The work tree (sources + Go module/build caches) runs to a couple of GB, so +# keep it inside the repo rather than under /tmp, which is often a small tmpfs. +# The Go module cache is written read-only; restore write permission before +# removing so cleanup never fails the script. +WORK_DIR="$(mktemp -d "${REPO_DIR}/.build-work.XXXXXX")" +trap 'chmod -R u+rwX "${WORK_DIR}" 2>/dev/null || true; rm -rf "${WORK_DIR}" 2>/dev/null || true' EXIT + +# Hermetic Go state by default: modules, build cache and on-demand toolchains +# all live under WORK_DIR, so a build never depends on (or pollutes) the +# host's Go dirs. CI overrides AUTONODE_GOPATH/AUTONODE_GOCACHE to persistent +# directories so modules and compiled packages survive across runs (only +# WORK_DIR is removed by the EXIT trap). +export GOPATH="${AUTONODE_GOPATH:-${WORK_DIR}/go}" +export GOCACHE="${AUTONODE_GOCACHE:-${WORK_DIR}/gocache}" +# Go's per-build scratch dir ($WORK) follows TMPDIR, not GOCACHE - point it +# at WORK_DIR too, or large builds can fill a small /tmp tmpfs. +export GOTMPDIR="${WORK_DIR}/gotmp" +mkdir -p "${GOTMPDIR}" +# -modcacherw keeps the module cache writable so cleanup and CI cache +# handling never trip over go's default read-only files (belt and braces +# with the chmod in the EXIT trap). +export GOFLAGS="-trimpath -mod=readonly -modcacherw" + +BUILT=0 +SKIPPED=0 + +# recipe_key ARGS... — cache key for one component: a hash of everything that +# determines its output except the sources themselves, which are pinned by +# tag and therefore immutable. +recipe_key() { + printf '%s\n' "schema=${STAMP_SCHEMA}" "go=${GO_VERSION}" \ + "goflags=${GOFLAGS}" "$@" | sha256sum | awk '{print $1}' +} + +# needs_build NAME KEY OUTPUTS... — decide whether NAME must be (re)built. +# Only a matching stamp with every staged output present counts as cached. +needs_build() { + local name="$1" key="$2" out ok=1 + shift 2 + [ "$(cat "${STAMP_DIR}/${name}" 2>/dev/null)" = "${key}" ] || ok=0 + for out in "$@"; do + [ -e "${OUT_DIR}/${out}" ] || ok=0 + done + if [ "${ok}" = 1 ]; then + echo " ${name}: cached, skipping" + SKIPPED=$((SKIPPED + 1)) + return 1 + fi + rm -f "${STAMP_DIR}/${name}" + return 0 +} + +# stamp NAME KEY — record a successful component build. +stamp() { + mkdir -p "${STAMP_DIR}" + printf '%s\n' "$2" > "${STAMP_DIR}/$1" + BUILT=$((BUILT + 1)) +} + +# check_binary PATH — sanity-check the produced ELF. Everything is built +# against glibc (static or dynamic) by construction; this catches accidental +# linkage against anything that would not resolve on the target (e.g. a +# scamper private .so if --disable-shared regressed) and logs what was built. +check_binary() { + local bin="$1" info + info="$(file -L "${bin}")" + case "${info}" in + *"statically linked"* | *"static-pie linked"*) ;; + *"dynamically linked"*) + if echo "${info}" | grep -q "ld-musl"; then + echo "ERROR: ${bin##*/} is musl-linked and would not run on the target." >&2 + exit 1 + fi + ;; + *) + echo "ERROR: ${bin} does not look like an executable: ${info}" >&2 + exit 1 + ;; + esac + echo " ${bin##*/}: ${info#*: }" +} + +# clone_repo NAME URL TAG — shallow-clone URL at TAG (once per NAME+TAG) and +# print the checkout directory. The clone keeps enough git metadata for the +# upstream version -ldflags (git describe / git log on the tagged commit). +clone_repo() { + local name="$1" url="$2" tag="$3" + local src="${WORK_DIR}/src/${name}-${tag}" + if [ ! -d "${src}" ]; then + git clone --quiet --depth 1 --branch "${tag}" "${url}" "${src}" >&2 + fi + echo "${src}" +} + +# short_commit SRC_DIR — the abbreviated hash of the tagged commit, used by the +# m-lab repos to stamp prometheusx.GitShortCommit (exposed as a metric label). +short_commit() { git -C "$1" log -1 --format=%h; } + +# go_build SRC_DIR PKG_DIR DEST CGO [LDFLAGS [GOTOOLCHAIN]] +# Build the Go main package at SRC_DIR/PKG_DIR into OUT_DIR/DEST. Runs from +# within PKG_DIR so nested modules (cmd/generate-schemas in ndt-server) build +# against their own go.mod. +go_build() { + local src="$1" pkg="$2" dest="$3" cgo="$4" ldflags="${5:-}" toolchain="${6:-}" + # Ship stripped: -s (symbol table) -w (DWARF); see the header comment. + ldflags="-s -w${ldflags:+ ${ldflags}}" + # cgo builds link dynamically anyway, so build them PIE for ASLR (lintian: + # hardening-no-pie); pure-Go builds stay internally-linked static, where + # -buildmode=pie does not apply. + local pie="" + [ "${cgo}" = "1" ] && pie="-buildmode=pie" + echo ">> building ${dest} (${src##*/} ${pkg})" + ( + cd "${src}/${pkg}" + CGO_ENABLED="${cgo}" \ + GOTOOLCHAIN="${toolchain:-auto}" \ + go build ${pie} -ldflags "${ldflags}" -o "${OUT_DIR}/${dest}" . + ) + check_binary "${OUT_DIR}/${dest}" +} + +# build_go DEST URL TAG PKG_DIR CGO [LDFLAGS_TEMPLATE [GOTOOLCHAIN]] +# Cached build of one Go component: skipped entirely (including the clone) +# when its recipe key matches the stamp of a previous run. LDFLAGS_TEMPLATE +# may reference @COMMIT@ / @COMMIT_FULL@, replaced with the short/full hash of +# the tagged commit — a pure function of TAG, hence not part of the key. +build_go() { + local dest="$1" url="$2" tag="$3" pkg="$4" cgo="$5" tmpl="${6:-}" toolchain="${7:-}" + local native="" + [ "${cgo}" = "1" ] && native="${NATIVE_TOOLCHAIN}" + local key + key="$(recipe_key "url=${url}" "tag=${tag}" "pkg=${pkg}" "cgo=${cgo}" \ + "ldflags=${tmpl}" "toolchain=${toolchain}" "native=${native}")" + needs_build "${dest}" "${key}" "${dest}" || return 0 + local src + src="$(clone_repo "$(basename "${url}" .git)" "${url}" "${tag}")" + local ldflags="${tmpl//@COMMIT@/$(short_commit "${src}")}" + ldflags="${ldflags//@COMMIT_FULL@/$(git -C "${src}" log -1 --format=%H)}" + go_build "${src}" "${pkg}" "${dest}" "${cgo}" "${ldflags}" "${toolchain}" + stamp "${dest}" "${key}" +} + +mkdir -p "${OUT_DIR}" +rm -f "${OUT_DIR}/.build-stamp" # pre-.stamps/ whole-script stamp, now unused + +# --- ndt-server (+ ndt7 schema generator) ----------------------------------- +# Upstream build.sh: CGO on (the ndt5 BBR code is cgo), stamped with the tag +# and short commit. cmd/generate-schemas is a nested Go module, so it is built +# from within its own directory. +NDT_SERVER_URL="https://github.com/m-lab/ndt-server.git" +build_go ndt-server "${NDT_SERVER_URL}" "${NDT_SERVER_VERSION}" . 1 \ + "-X github.com/m-lab/ndt-server/version.Version=${NDT_SERVER_VERSION} \ + -X github.com/m-lab/go/prometheusx.GitShortCommit=@COMMIT@" \ + "${NDT_GO_TOOLCHAIN}" +build_go generate-schemas-ndt7 "${NDT_SERVER_URL}" "${NDT_SERVER_VERSION}" \ + cmd/generate-schemas 1 "" "${NDT_GO_TOOLCHAIN}" + +# --- heartbeat (from m-lab/locate) ------------------------------------------- +build_go heartbeat https://github.com/m-lab/locate.git "${LOCATE_VERSION}" \ + cmd/heartbeat 0 \ + "-X github.com/m-lab/go/prometheusx.GitShortCommit=@COMMIT@" + +# --- uuid-annotator (+ annotation2 schema generator, older pin) --------------- +UUID_ANNOTATOR_URL="https://github.com/m-lab/uuid-annotator.git" +build_go uuid-annotator "${UUID_ANNOTATOR_URL}" "${UUID_ANNOTATOR_VERSION}" . 0 \ + "-X github.com/m-lab/go/prometheusx.GitShortCommit=@COMMIT@" + +# The IPInfo AS-names CSV the annotator reads via -asname.url. The image +# shipped it at /data and pointed at it with a container ENV; systemd units +# have no image ENVs, so the package ships the file and passes the flag. +key="$(recipe_key "component=asnames" "url=${UUID_ANNOTATOR_URL}" \ + "tag=${UUID_ANNOTATOR_VERSION}")" +if needs_build asnames.ipinfo.csv "${key}" asnames.ipinfo.csv; then + src="$(clone_repo uuid-annotator "${UUID_ANNOTATOR_URL}" "${UUID_ANNOTATOR_VERSION}")" + install -D -m 0644 "${src}/data/asnames.ipinfo.csv" "${OUT_DIR}/asnames.ipinfo.csv" + echo " asnames.ipinfo.csv: $(wc -c <"${OUT_DIR}/asnames.ipinfo.csv" | tr -d ' ') bytes" + stamp asnames.ipinfo.csv "${key}" +fi + +build_go generate-schemas-annotation2 "${UUID_ANNOTATOR_URL}" \ + "${UUID_ANNOTATOR_SCHEMA_VERSION}" cmd/generate-schemas 0 + +# --- jostler ------------------------------------------------------------------- +# Upstream also stamps main.Version (git describe, i.e. the tag) and +# main.GitCommit (the full hash). +build_go jostler https://github.com/m-lab/jostler.git "${JOSTLER_VERSION}" \ + cmd/jostler 0 \ + "-X github.com/m-lab/go/prometheusx.GitShortCommit=@COMMIT@ \ + -X main.Version=${JOSTLER_VERSION} \ + -X main.GitCommit=@COMMIT_FULL@" + +# --- traceroute-caller (+ traceroute schema generator + scamper) --------------- +TRACEROUTE_URL="https://github.com/m-lab/traceroute-caller.git" +build_go traceroute-caller "${TRACEROUTE_URL}" "${TRACEROUTE_VERSION}" . 0 \ + "-X github.com/m-lab/go/prometheusx.GitShortCommit=@COMMIT@" +build_go generate-schemas-traceroute "${TRACEROUTE_URL}" "${TRACEROUTE_VERSION}" \ + cmd/generate-schemas 0 + +# scamper: build the exact tarball the traceroute-caller image vendors. +# --disable-shared statically links scamper's internal libraries into the +# binary, so the package can ship the single file with no private .so deps. +# --disable-privsep drops scamper's built-in privilege separation (chroot to +# /var/empty + setuid), which cannot work - by design - inside the +# traceroute-caller unit's systemd sandbox (read-only /var, no CAP_SYS_CHROOT +# or CAP_SETUID, chroot filtered) and is redundant there: the unit itself +# confines scamper, and raw sockets come from the inherited CAP_NET_RAW +# ambient capability. (The container image instead granted scamper +# cap_sys_chroot,cap_setuid,cap_setgid file capabilities to let privsep run.) +# OpenSSL is deliberately not linked: configure would silently pick it up if +# the dev headers happen to be installed, making the build host-dependent. The +# image's scamper was built without it, and traceroute-caller does not use the +# TLS probes. There is no --without-openssl switch (AX_CHECK_OPENSSL), so point +# the search at an empty directory instead. +SCAMPER_CONFIGURE_FLAGS="--disable-shared --disable-privsep --with-openssl=/nonexistent" +key="$(recipe_key "component=scamper" "dist=${SCAMPER_DIST}" \ + "vendored-in=${TRACEROUTE_VERSION}" "configure=${SCAMPER_CONFIGURE_FLAGS}" \ + "native=${NATIVE_TOOLCHAIN}")" +if needs_build scamper "${key}" scamper; then + src="$(clone_repo traceroute-caller "${TRACEROUTE_URL}" "${TRACEROUTE_VERSION}")" + echo ">> building scamper (${SCAMPER_DIST}, vendored in traceroute-caller)" + scamper_build="${WORK_DIR}/scamper" + scamper_prefix="${WORK_DIR}/scamper-install" + mkdir -p "${scamper_build}" + tar -xzf "${src}/third_party/scamper/${SCAMPER_DIST}.tar.gz" -C "${scamper_build}" + ( + cd "${scamper_build}/${SCAMPER_DIST}" + chmod +x ./configure + # shellcheck disable=SC2086 # deliberate word splitting of the flags + ./configure --quiet --prefix="${scamper_prefix}" ${SCAMPER_CONFIGURE_FLAGS} + make --quiet -j"$(nproc)" >/dev/null + make --quiet install >/dev/null + ) + install -m 0755 "${scamper_prefix}/bin/scamper" "${OUT_DIR}/scamper" + check_binary "${OUT_DIR}/scamper" + stamp scamper "${key}" +fi + +# --- autojoin-register ---------------------------------------------------------- +build_go autojoin-register https://github.com/m-lab/autojoin.git \ + "${AUTOJOIN_VERSION}" cmd/register 0 \ + "-X main.Version=${AUTOJOIN_VERSION}" + +# --- node-exporter --------------------------------------------------------------- +build_go node-exporter https://github.com/prometheus/node_exporter.git \ + "${NODE_EXPORTER_VERSION}" . 0 \ + "-X github.com/prometheus/common/version.Version=${NODE_EXPORTER_VERSION#v} \ + -X github.com/prometheus/common/version.Revision=@COMMIT_FULL@ \ + -X github.com/prometheus/common/version.Branch=HEAD" + +echo +echo "Components: ${BUILT} built, ${SKIPPED} cached; staged in ${OUT_DIR}" diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml new file mode 100644 index 0000000..4334ae3 --- /dev/null +++ b/.github/workflows/build-deb.yml @@ -0,0 +1,159 @@ +# Build the mlab-node Debian package. +# +# Every push/PR builds the .deb for amd64 and arm64 — natively on +# ubuntu-24.04-arm runners for the latter, no QEMU — inside a debian:bookworm +# container (the oldest supported target: its glibc sets the package's libc6 +# floor, so the .deb installs on bookworm and anything newer) and uploads +# them as CI artifacts. Pushing a v* tag additionally creates a GitHub +# Release with the .debs attached, versioned from the tag. +# +# Caching: binaries/ carries per-component stamps (see +# .build/build-binaries.sh), so a restored cache rebuilds only the components +# whose pins or build flags changed; the Go module/build caches make even a +# full rebuild incremental. +name: build-deb + +on: + pull_request: + push: + branches: [main] + tags: ['v*'] + workflow_dispatch: + +concurrency: + group: build-deb-${{ github.ref }} + cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/') }} + +jobs: + build: + name: build (${{ matrix.arch }}) + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + arch: amd64 + - runner: ubuntu-24.04-arm + arch: arm64 + runs-on: ${{ matrix.runner }} + container: debian:bookworm + # Container jobs default to sh; the version-stamping step uses bashisms. + defaults: + run: + shell: bash + env: + DEBEMAIL: support@measurementlab.net + DEBFULLNAME: Measurement Lab + steps: + - name: Install build dependencies + run: | + apt-get update + apt-get install -y --no-install-recommends \ + build-essential debhelper devscripts lintian git file \ + ca-certificates curl + + - uses: actions/checkout@v4 + + # ${{ github.workspace }} resolves to the HOST path in container jobs; + # $GITHUB_WORKSPACE holds the container-side mount, which is what the + # build script (and the actions/cache 'path: .go-cache' below) need. + - name: Point Go caches into the workspace + run: | + echo "AUTONODE_GOPATH=${GITHUB_WORKSPACE}/.go-cache/gopath" >> "$GITHUB_ENV" + echo "AUTONODE_GOCACHE=${GITHUB_WORKSPACE}/.go-cache/gocache" >> "$GITHUB_ENV" + + # Go comes from setup-go, not apt: bookworm's golang-go is far too old + # for the pinned components. Its built-in cache is disabled because it + # keys on go.sum files, which this repo does not have. + - uses: actions/setup-go@v5 + with: + go-version: stable + cache: false + + - name: Compute cache fingerprint + id: fp + run: echo "go=$(go version | awk '{print $3}')" >> "$GITHUB_OUTPUT" + + # Stale restores are useful (restore-keys): the per-component stamps let + # the build script skip everything whose recipe did not change. + - name: Cache built component binaries + uses: actions/cache@v4 + with: + path: binaries + key: binaries-${{ matrix.arch }}-${{ steps.fp.outputs.go }}-${{ hashFiles('.build/build-binaries.sh') }} + restore-keys: | + binaries-${{ matrix.arch }}-${{ steps.fp.outputs.go }}- + binaries-${{ matrix.arch }}- + + - name: Cache Go modules and build cache + uses: actions/cache@v4 + with: + path: .go-cache + key: go-${{ matrix.arch }}-${{ steps.fp.outputs.go }}-${{ hashFiles('.build/build-binaries.sh') }} + restore-keys: | + go-${{ matrix.arch }}-${{ steps.fp.outputs.go }}- + go-${{ matrix.arch }}- + + # Tag builds get the tag as their version; anything else gets a + # ~dev..g suffix, which sorts before the plain version so + # snapshots upgrade cleanly to the eventual release. + - name: Set package version + run: | + if [ "${GITHUB_REF_TYPE}" = "tag" ]; then + dch -b --newversion "${GITHUB_REF_NAME#v}" \ + --distribution unstable "Release ${GITHUB_REF_NAME}." + else + base="$(dpkg-parsechangelog -SVersion)" + dch -b --newversion "${base}~dev.${GITHUB_RUN_NUMBER}.g${GITHUB_SHA::7}" \ + --distribution UNRELEASED "CI snapshot build." + fi + dpkg-parsechangelog -SVersion + + # -d: the golang-go build-dep is satisfied by setup-go, invisible to dpkg. + - name: Build package + run: dpkg-buildpackage -us -uc -b -d + + - name: Lintian + run: lintian --fail-on error --tag-display-limit 0 ../mlab-node_*.changes + + # The postinst guards every systemd interaction on /run/systemd/system, + # so installing in the build container exercises the maintainer scripts + # (sysusers, tmpfiles, enable) without an init system. Installing on + # bookworm itself proves the computed libc6 floor is satisfiable on the + # oldest supported target. + - name: Install smoke test + run: | + apt-get install -y ../mlab-node_*.deb + dpkg -V mlab-node + echo "Depends: $(dpkg-deb -f ../mlab-node_*.deb Depends)" + + - name: Collect artifacts + run: mkdir -p dist && mv ../mlab-node_* dist/ + + - uses: actions/upload-artifact@v4 + with: + name: mlab-node-deb-${{ matrix.arch }} + path: dist/ + if-no-files-found: error + + release: + if: github.ref_type == 'tag' + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: mlab-node-deb-* + merge-multiple: true + path: dist + + - name: Create GitHub Release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "${GITHUB_REF_NAME}" dist/*.deb \ + --repo "${GITHUB_REPOSITORY}" \ + --title "${GITHUB_REF_NAME}" \ + --generate-notes diff --git a/.gitignore b/.gitignore index 2eea525..83e0c6e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,22 @@ -.env \ No newline at end of file +.env +.DS_Store + +# Binaries built from the pinned upstream sources at build time (see +# .build/build-binaries.sh). Not checked in. +/binaries/ +/.build-work.* + +# Persistent Go caches used by CI (AUTONODE_GOPATH/AUTONODE_GOCACHE). +/.go-cache/ + +# Debian build artifacts. +/debian/mlab-node/ +/debian/.debhelper/ +/debian/files +/debian/*.debhelper +/debian/*.debhelper.log +/debian/*.substvars +/debian/debhelper-build-stamp +*.deb +*.buildinfo +*.changes diff --git a/README.md b/README.md index c343eaf..5309d5a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,117 @@ -# autonode +# autonode (mlab-node) -Documentation on how to set up a host-managed M-Lab server is located in [the -Wiki of this -repository](https://github.com/m-lab/autonode/wiki/Host%E2%80%90managed-Deployments). +Native Debian packaging of the M-Lab "autonode" Network Diagnostic Tool (NDT) +measurement stack. The components run as hardened **systemd** services, confined +with systemd's own sandboxing directives — there is no Docker at build or run +time. + +This replaces the previous Docker Compose deployment. Documentation on +host-managed M-Lab servers is in the +[Wiki](https://github.com/m-lab/autonode/wiki/Host%E2%80%90managed-Deployments). + +## What's in the package + +The `mlab-node` package installs and runs, as systemd services: + +| Service | Unit | +|---|---| +| Node registration (Autojoin API) | `mlab-node-register.service` | +| NDT7 measurement server | `mlab-node-ndt-server.service` | +| Heartbeat (Locate API) | `mlab-node-heartbeat.service` | +| UUID annotator | `mlab-node-uuid-annotator.service` | +| Jostler (GCS archival) | `mlab-node-jostler.service` | +| Traceroute caller (scamper) | `mlab-node-traceroute-caller.service` | +| Prometheus node exporter | `mlab-node-node-exporter.service` | + +plus one-shot units for schema generation, the UUID prefix, BBR, and external-IP +metadata. The whole stack is controlled atomically through `mlab-node.target`; +`register-node` is the keystone — if it goes down, the entire stack is brought +down with it. + +The component binaries are built from the pinned upstream sources at package +build time (see `.build/build-binaries.sh`) — the same version tags the +container images were built from. + +## File layout + +| Path | Contents | +|---|---| +| `/usr/lib/mlab/` | component binaries, helper scripts, package-owned constants (`static.env`) | +| `/etc/mlab/mlab-node.env` | configuration (non-secret) | +| `/etc/mlab/api-key.env` | M-Lab API key (`0640 root:mlab-node`) | +| `/etc/mlab/locate/verify.pub` | Locate token verification key | +| `/var/lib/mlab/node/` | registration data + service-account credential | +| `/var/lib/mlab/schemas/` | generated JSON schemas + uuid.prefix | +| `/var/lib/mlab/resultsdir/` | measurement results awaiting upload | +| `/var/lib/mlab/autocert/` | Let's Encrypt certificate cache | +| `/run/mlab/sockets/` | tcpinfo / ipservice unix sockets | +| `/usr/share/mlab/html/` | static content served by ndt-server | + +## Building the package + +**CI is the canonical build**: every push runs the +[`build-deb` workflow](.github/workflows/build-deb.yml), which builds the +package for **amd64 and arm64** (natively, on `ubuntu-24.04-arm` runners for +the latter) in a `debian:bookworm` container — bookworm's glibc sets the +package's `libc6` floor, so the .deb installs on Debian 12 and anything +newer — and uploads them as workflow artifacts. Pushing a `v*` tag creates a +GitHub Release with the .debs attached; Releases are what production machines +should install. + +Local builds still work and use the same script. Requires +`dpkg-buildpackage`, debhelper, a recent Go, `git`, `file`, and a C toolchain +(for scamper and the cgo builds), plus network access to clone the pinned +sources and fetch Go modules: + +```sh +dpkg-buildpackage -us -uc -b +``` + +(Add `-d` if Go is installed from upstream tarballs rather than the +`golang-go` package.) Note that a locally built .deb inherits the host's +glibc as its `libc6` dependency floor — a package built on a newer distro may +not install on bookworm; use the CI artifacts for anything that leaves your +machine. + +The build clones each component's repository at its pinned tag, builds the +binaries (replicating the upstream image builds' flags, plus `-s -w` +stripping), and stages them under `binaries/`. No container images or Docker +are involved. Each component is cached with a recipe-keyed stamp under +`binaries/.stamps/`, so re-builds only recompile components whose version pin +or build flags changed; delete `binaries/` to force a full rebuild. + +## Installing and configuring + +```sh +sudo apt install ./mlab-node_1.0.0_amd64.deb + +# Configure the node, then add the API key. +sudoedit /etc/mlab/mlab-node.env +sudoedit /etc/mlab/api-key.env + +# Start the whole stack (it is also enabled to start on boot). +sudo systemctl start mlab-node.target +``` + +Required settings in `/etc/mlab/mlab-node.env`: `ORGANIZATION`, `IATA`, +`INTERFACE_NAME`, `INTERFACE_MAXRATE`, `UPLINK`, `IPV4`, `TYPE` (and `IPV6` if +the node is reachable over IPv6). `API_KEY` goes in `/etc/mlab/api-key.env`. + +## Operating + +```sh +systemctl status mlab-node.target +systemctl list-dependencies mlab-node.target +journalctl -u mlab-node-ndt-server -f + +# Stop / restart the entire stack. +sudo systemctl stop mlab-node.target +sudo systemctl restart mlab-node.target +``` + +## Prometheus metrics + +Exposed on the host (all `network_mode: host` equivalents are now native): +9990 ndt-server, 9991 jostler, 9992 uuid-annotator, 9993 heartbeat, +9994 traceroute-caller, 9995 node-exporter. ndt-server serves measurements on +80 (cleartext) and 443 (TLS). diff --git a/config/api-key.env b/config/api-key.env new file mode 100644 index 0000000..46f7d6b --- /dev/null +++ b/config/api-key.env @@ -0,0 +1,3 @@ +# M-Lab API key for this node. Keep this file secret. +# Installed as 0640 root:mlab-node. Set the value provided by M-Lab below. +API_KEY= diff --git a/config/locate/verify.pub b/config/locate/verify.pub new file mode 100644 index 0000000..a2ea853 --- /dev/null +++ b/config/locate/verify.pub @@ -0,0 +1 @@ +{"use":"sig","kty":"OKP","kid":"locate_20200409","crv":"Ed25519","alg":"EdDSA","x":"1tS1d-dd2B-VRBTWzOaq7zUngKDyV409K-o42LN2nx8"} diff --git a/config/mlab-node.env b/config/mlab-node.env new file mode 100644 index 0000000..f0d5590 --- /dev/null +++ b/config/mlab-node.env @@ -0,0 +1,39 @@ +# Configuration for the M-Lab node (mlab-node) services. +# This file is sourced by the systemd units as an EnvironmentFile. Edit the +# values below to match your deployment, then start the stack with: +# systemctl start mlab-node.target +# +# The API key is kept separately in /etc/mlab/api-key.env. + +# ORGANIZATION is the organization name assigned by M-Lab to your deployment. +ORGANIZATION= + +# IATA is the 3-letter, lowercase IATA code of the nearest international +# airport. Estimate it from your latitude/longitude: +# https://autojoin-dot-mlab-autojoin.appspot.com/autojoin/v0/lookup?lat=&lon= +IATA= + +# PROBABILITY is the probability (0-1) that this node is selected by M-Lab's +# load balancer. Default is 0.1 (10%). +PROBABILITY=0.1 + +# INTERFACE_NAME is the network interface to which INTERFACE_MAXRATE applies. +INTERFACE_NAME= + +# UPLINK is the uplink speed, an integer followed by "g" (e.g. "1g", "10g", +# "25g"). Minimum is 1g; 10g or greater is recommended. +UPLINK= + +# INTERFACE_MAXRATE is the rate (bit/s) above which ndt-server refuses new +# connections. Recommended: 1G -> 150000000, 10G -> 7000000000. +INTERFACE_MAXRATE= + +# IPV4 is the external IPv4 address this node is reachable at. Required. +IPV4= + +# IPV6 is the external IPv6 address this node is reachable at. Optional; set it +# to make the node accessible over IPv6. +IPV6= + +# TYPE is "physical" (bare-metal dedicated to M-Lab) or "virtual" (a VM). +TYPE= diff --git a/config/modules-load.d/mlab-node.conf b/config/modules-load.d/mlab-node.conf new file mode 100644 index 0000000..da42f81 --- /dev/null +++ b/config/modules-load.d/mlab-node.conf @@ -0,0 +1,3 @@ +# ndt-server uses the BBR congestion control algorithm, which requires the +# tcp_bbr kernel module to be loaded at boot. +tcp_bbr diff --git a/config/static.env b/config/static.env new file mode 100644 index 0000000..60350ac --- /dev/null +++ b/config/static.env @@ -0,0 +1,8 @@ +# Package-owned constants for the mlab-node services. Installed under +# /usr/lib/mlab (not a conffile) so package upgrades can change them without +# conffile merge prompts. The units load this file before the operator-edited +# /etc/mlab/mlab-node.env, so values set there take precedence if you must +# override these. +PROJECT=mlab-autojoin +LOCATE_URL=locate.measurementlab.net +AUTH_URL=auth.mlab-oti.measurementlab.net diff --git a/config/sysctl.d/90-mlab-node.conf b/config/sysctl.d/90-mlab-node.conf new file mode 100644 index 0000000..0985521 --- /dev/null +++ b/config/sysctl.d/90-mlab-node.conf @@ -0,0 +1,4 @@ +# Enable BBR congestion control for ndt-server. Requires the tcp_bbr module +# (loaded via /usr/lib/modules-load.d/mlab-node.conf). +net.core.default_qdisc = fq +net.ipv4.tcp_congestion_control = bbr diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..710ad8e --- /dev/null +++ b/debian/changelog @@ -0,0 +1,8 @@ +mlab-node (1.0.0) unstable; urgency=medium + + * Initial release: native Debian packaging of the M-Lab autonode stack. + Replaces the Docker Compose deployment with hardened systemd services. + Component binaries are built from the pinned upstream sources at build + time; no Docker or container images are involved at build or run time. + + -- Measurement Lab Mon, 29 Jun 2026 00:00:00 +0000 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..2028b16 --- /dev/null +++ b/debian/control @@ -0,0 +1,30 @@ +Source: mlab-node +Section: net +Priority: optional +Maintainer: Measurement Lab +Build-Depends: + debhelper-compat (= 13), + file, + golang-go (>= 2:1.21~), + git, + ca-certificates, + gcc, + make, + libc6-dev, +Standards-Version: 4.6.2 +Homepage: https://github.com/m-lab/autonode +Rules-Requires-Root: no + +Package: mlab-node +Architecture: amd64 arm64 +Depends: + ${shlibs:Depends}, + ${misc:Depends}, + curl, + kmod, +Description: M-Lab node (autonode) measurement stack + Native systemd packaging of the M-Lab "autonode" Network Diagnostic Tool + (NDT) measurement platform, replacing the former Docker Compose deployment. + . + It runs ndt-server and its sidecar services - node registration, heartbeat, + uuid-annotator, jostler, traceroute-caller and a Prometheus node exporter. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..909856d --- /dev/null +++ b/debian/copyright @@ -0,0 +1,33 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: autonode +Source: https://github.com/m-lab/autonode + +Files: * +Copyright: 2025-2026 Measurement Lab +License: Apache-2.0 + +Files: binaries/* +Copyright: 2025-2026 Measurement Lab and the Prometheus Authors +Comment: + These binaries are built at package build time from the pinned upstream + sources by .build/build-binaries.sh. They are not part of the source in this + repository. Each component is licensed under its own upstream license; see + the respective M-Lab, Prometheus and scamper projects (scamper, vendored in + m-lab/traceroute-caller, is GPL-2 licensed). +License: Apache-2.0 + +License: Apache-2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + https://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian systems, the full text of the Apache License version 2.0 can be + found in the file /usr/share/common-licenses/Apache-2.0. diff --git a/debian/mlab-node.install b/debian/mlab-node.install new file mode 100644 index 0000000..1ccf5f7 --- /dev/null +++ b/debian/mlab-node.install @@ -0,0 +1,43 @@ +# Component binaries (built from the pinned sources by .build/build-binaries.sh). +binaries/ndt-server usr/lib/mlab +binaries/heartbeat usr/lib/mlab +binaries/uuid-annotator usr/lib/mlab +binaries/jostler usr/lib/mlab +binaries/traceroute-caller usr/lib/mlab +binaries/scamper usr/lib/mlab +binaries/autojoin-register usr/lib/mlab +binaries/node-exporter usr/lib/mlab +binaries/generate-schemas-ndt7 usr/lib/mlab +binaries/generate-schemas-annotation2 usr/lib/mlab +binaries/generate-schemas-traceroute usr/lib/mlab + +# Helper scripts. +scripts/gen-uuid-prefix usr/lib/mlab +scripts/write-metadata usr/lib/mlab +scripts/wait-socket usr/lib/mlab + +# systemd units. Installed under the canonical /usr path: on merged-/usr +# systems, shipping files under the aliased /lib is a policy violation +# (lintian: aliased-location). +systemd/*.service usr/lib/systemd/system +systemd/*.target usr/lib/systemd/system + +# Configuration (files under /etc are registered as conffiles automatically). +config/mlab-node.env etc/mlab +config/api-key.env etc/mlab +config/locate/verify.pub etc/mlab/locate + +# Package-owned constants (not a conffile; loaded by the units before the +# operator-edited /etc/mlab/mlab-node.env). +config/static.env usr/lib/mlab + +# Kernel module + sysctl drop-ins for BBR. +config/modules-load.d/mlab-node.conf usr/lib/modules-load.d +config/sysctl.d/90-mlab-node.conf usr/lib/sysctl.d + +# Static content served by ndt-server. +html/index.html usr/share/mlab/html + +# IPInfo AS-names CSV (from the uuid-annotator repo), read by the +# uuid-annotator via -asname.url. +binaries/asnames.ipinfo.csv usr/share/mlab diff --git a/debian/mlab-node.lintian-overrides b/debian/mlab-node.lintian-overrides new file mode 100644 index 0000000..aa11da0 --- /dev/null +++ b/debian/mlab-node.lintian-overrides @@ -0,0 +1,3 @@ +# The pure-Go binaries are statically linked by nature (internal linking, no +# cgo); this is how upstream builds and ships them. +statically-linked-binary [usr/lib/mlab/*] diff --git a/debian/mlab-node.postinst b/debian/mlab-node.postinst new file mode 100644 index 0000000..7bb6a4b --- /dev/null +++ b/debian/mlab-node.postinst @@ -0,0 +1,25 @@ +#!/bin/sh +set -e + +#DEBHELPER# + +# On upgrade ($2 is the previously-installed version), restart the stack so +# it picks up the new binaries and unit files. The explicit daemon-reload +# mirrors dh_installsystemd's own restart-after-upgrade snippet: debhelper +# 13.31 defers reload to a dpkg file trigger that only fires after postinst, +# so without it the restart would use the stale in-memory units. Every +# component is PartOf=mlab-node.target and pulled in by it, so restarting the +# target cycles the whole stack. On first install ($2 is empty) do nothing: +# the target is enabled but must not start before the operator configures +# /etc/mlab/*.env. +if [ "$1" = "configure" ] && [ -n "$2" ] && [ -d /run/systemd/system ]; then + systemctl --system daemon-reload >/dev/null || true + # --no-block: queue the restart and return, instead of holding dpkg for + # the minutes the stack can take to come up (register-ready gates every + # service on a registration retry loop). Startup failures land in the + # journal, not the apt transcript. systemctl is called directly because + # deb-systemd-invoke cannot forward --no-block; the policy-rc.d layer it + # provides matters in chroots, which the /run/systemd/system guard + # already excludes. + systemctl --no-block restart mlab-node.target >/dev/null || true +fi diff --git a/debian/mlab-node.sysusers b/debian/mlab-node.sysusers new file mode 100644 index 0000000..678bca4 --- /dev/null +++ b/debian/mlab-node.sysusers @@ -0,0 +1,4 @@ +# Shared system user/group for the M-Lab node services. A single identity owns +# the service-account credential and the shared unix sockets, which are the only +# things the components share. node-exporter runs as a DynamicUser instead. +u mlab-node - "M-Lab node services" /var/lib/mlab /usr/sbin/nologin diff --git a/debian/mlab-node.tmpfiles b/debian/mlab-node.tmpfiles new file mode 100644 index 0000000..834880f --- /dev/null +++ b/debian/mlab-node.tmpfiles @@ -0,0 +1,20 @@ +# Persistent state under /var/lib/mlab and runtime dirs under /run/mlab. +# setgid (2xxx) so files and sockets created here inherit the mlab-node group. + +# Persistent state. +d /var/lib/mlab 0755 mlab-node mlab-node - +d /var/lib/mlab/node 2750 mlab-node mlab-node - +d /var/lib/mlab/schemas 2755 mlab-node mlab-node - +d /var/lib/mlab/resultsdir 2755 mlab-node mlab-node - +d /var/lib/mlab/resultsdir/ndt 2755 mlab-node mlab-node - +d /var/lib/mlab/autocert 2750 mlab-node mlab-node - + +# Runtime (tmpfs, recreated each boot). +d /run/mlab 0755 mlab-node mlab-node - +d /run/mlab/sockets 2770 mlab-node mlab-node - +d /run/mlab/metadata 2755 mlab-node mlab-node - + +# Restrict the API key conffile to the service group (shipped 0644 root:root by +# dpkg). Applied on install by the debhelper tmpfiles snippet, which runs after +# the sysusers snippet has created the mlab-node group, and re-applied each boot. +z /etc/mlab/api-key.env 0640 root mlab-node - diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..66fdf05 --- /dev/null +++ b/debian/rules @@ -0,0 +1,40 @@ +#!/usr/bin/make -f + +%: + dh $@ + +# dh_installsysusers is not part of the default dh sequence on debhelper 13.14 +# (unlike dh_installtmpfiles), so invoke it explicitly to create the mlab-node +# system user from debian/mlab-node.sysusers. Running it here (after dh_install, +# before dh_installtmpfiles) orders the postinst so the user exists before the +# tmpfiles directories are chowned to it. +execute_after_dh_install: + dh_installsysusers + +# Build the component binaries from the pinned upstream sources. The script +# owns its own caching: each component carries a recipe-keyed stamp under +# binaries/.stamps/, so only components whose pin or build flags changed are +# rebuilt. +override_dh_auto_build: + ./.build/build-binaries.sh + +# Install the target so it is enabled (auto-start on boot) but not started at +# install time - the operator must configure /etc/mlab/*.env first. The other +# units carry no [Install] section (dh_installsystemd only enables units that +# have one) and are pulled in by the target, so they are only registered for +# daemon-reload, not enabled or started individually. +# +# --no-stop-on-upgrade keeps the old services running while an upgrade +# unpacks; the postinst then restarts mlab-node.target (upgrades only) so the +# stack picks up the new binaries. Without it, prerm stops every unit and +# --no-start would leave the node down after the upgrade. +override_dh_installsystemd: + dh_installsystemd --no-start --no-stop-on-upgrade + +# The Go binaries are linked with -s -w (see .build/build-binaries.sh), which +# leaves dh_strip nothing to do except scamper (C). Strip it, but skip the +# dbgsym package - nothing would consume it. dwz does not apply to Go. +override_dh_strip: + dh_strip --no-automatic-dbgsym + +override_dh_dwz: diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..89ae9db --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index edcedcf..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,348 +0,0 @@ -services: - # Register this node with the Autojoin API. A successful registration will - # assign a DNS name for this node based on the public IPs, their ASN, - # organiztion, and IATA code. As well, Register distributes other credentials - # needed to upload and archive measurement data. The register command contacts - # the Autojoin API every hour. - register-node: - # NOTE: register should not be restarted on failure. - image: measurementlab/autojoin-register:v0.2.13 - profiles: [check-config, ndt] - pull_policy: always - network_mode: host - volumes: - - ./autonode:/autonode - restart: always - command: - - -endpoint=https://autojoin-dot-${PROJECT}.appspot.com/autojoin/v0/node/register - - -key=${API_KEY} - - -service=ndt - - -iata=${IATA} - - -output=/autonode - - -healthcheck-addr=:8001 - - -ports=9990,9991,9992,9993,9994,9995 - - -probability=${PROBABILITY} - - -ipv4=${IPV4} - - -ipv6=${IPV6} - - -type=${TYPE} - - -uplink=${UPLINK} - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8001/ready"] - interval: 3s - timeout: 5s - retries: 10 - - # ndt-server is the public facing measurement service. Only ndt7 is enabled. - ndt-server: - image: measurementlab/ndt-server:v0.25.3 - profiles: [ndt] - network_mode: host - cap_add: - - NET_BIND_SERVICE - volumes: - - ./certs:/certs - - ./html:/html - - ./schemas:/schemas - - ./resultsdir:/resultsdir - - ./autonode:/autonode - - ./autocert:/autocert - depends_on: - generate-schemas-ndt7: - condition: service_completed_successfully - check-bbr: - condition: service_completed_successfully - register-node: - condition: service_healthy - restart: always - # NOTE: All containers will use the host network. All the following ports - # must not be in use. - ports: - # ndt-server TLS and non-TLS ports. - - target: 443 - published: 443 - protocol: tcp - - target: 80 - published: 80 - protocol: tcp - # ndt-server prometheus. - - target: 9990 - published: 9990 - protocol: tcp - # jostler prometheus. - - target: 9991 - published: 9991 - protocol: tcp - # annotator prometheus. - - target: 9992 - published: 9992 - protocol: tcp - # heartbeat prometheus. - - target: 9993 - published: 9993 - protocol: tcp - # traceroute-caller prometheus. - - target: 9994 - published: 9994 - protocol: tcp - # Generate the UUID prefix before starting ndt-server. This runs on every - # container start (including restart:always restarts after host reboot), - # ensuring a fresh prefix with the current boot time. The prefix uses btime - # from /proc/stat, which is a deterministic integer — no coordination file - # or separate container needed. - entrypoint: - - /bin/sh - - -c - - | - btime=$$(awk '/^btime / {print $$2}' /proc/stat) && - printf '%s_%s' "$$(hostname)" "$$btime" > /schemas/uuid.prefix && - exec /ndt-server "$$@" - - -- - command: - - -uuid-prefix-file=/schemas/uuid.prefix - - -datadir=/resultsdir/ndt - - -ndt7_addr=:443 - - -ndt7_addr_cleartext=:80 - # Allow ndt7 data to be autoloaded. - - -compress-results=false - # TODO(host): confirm device name. - - -txcontroller.device=${INTERFACE_NAME} - - -txcontroller.max-rate=${INTERFACE_MAXRATE} - - -prometheusx.listen-address=:9990 - # Add server metadata. - - -label=type=${TYPE} - - -label=deployment=byos - - -label=managed=none - - -label=loadbalanced=false - - -label=org=${ORGANIZATION} - - -label=external-ip=${IPV4} - - -label=external-ipv6=${IPV6} - # Effectively disable ndt5. - - -ndt5_addr=127.0.0.1:3002 - - -ndt5_ws_addr=127.0.0.1:3001 - - -tcpinfo.eventsocket=/schemas/events.sock - - -autocert.enabled=true - - -autocert.hostname=@/autonode/hostname - - -token.machine=@/autonode/hostname - - -token.verify-key=/locate/verify.pub - - -ndt7.token.required=true - configs: - - source: locate-verify-key - target: /locate/verify.pub - healthcheck: - test: ["CMD", "wget", "-O/dev/null", "-q", "http://localhost:80/"] - interval: 3s - timeout: 5s - retries: 10 - - # heartbeat reports health and liveness messages to M-Lab's Locate API every - # 10sec. The Locate API is responsible for directing clients to nearby healthy - # servers. - heartbeat: - image: measurementlab/heartbeat:v0.19.1 - profiles: [ndt] - volumes: - - ./autonode:/autonode - depends_on: - ndt-server: - condition: service_started - register-node: - condition: service_healthy - restart: always - command: - - -prometheusx.listen-address=:9993 - - -experiment=ndt - - -hostname=@/autonode/hostname - - -registration-url=file:///autonode/registration.json - - -heartbeat-url=wss://${LOCATE_URL}/v2/platform/heartbeat-jwt - - -api-key=${API_KEY} - - -token-exchange-url=https://${AUTH_URL}/v0/token/autojoin - - -services=ndt/ndt7=ws:///ndt/v7/download,ws:///ndt/v7/upload,wss:///ndt/v7/download,wss:///ndt/v7/upload - network_mode: host - - # uuid-annotator records network and geographic annotations based on the - # client and server IP so each NDT measurement is annotated in BigQuery. - uuid-annotator: - image: measurementlab/uuid-annotator:v0.5.10 - profiles: [ndt] - volumes: - - ./resultsdir:/resultsdir - - ./schemas:/schemas - - ./certs:/certs - - ./autonode:/autonode - depends_on: - ndt-server: - condition: service_started - register-node: - condition: service_healthy - network_mode: host - environment: - - GOOGLE_APPLICATION_CREDENTIALS=/autonode/service-account-autojoin.json - restart: always - # NOTE: all database URLs are required. - command: - - -prometheusx.listen-address=:9992 - - -datadir=/resultsdir/ndt/annotation2 - - -tcpinfo.eventsocket=/schemas/events.sock - - -ipservice.sock=/schemas/uuid-annotator.sock - - -maxmind.url=gs://downloader-${PROJECT}/Maxmind/current/GeoLite2-City.tar.gz - - -routeview-v4.url=gs://downloader-${PROJECT}/RouteViewIPv4/current/routeview.pfx2as.gz - - -routeview-v6.url=gs://downloader-${PROJECT}/RouteViewIPv6/current/routeview.pfx2as.gz - - -siteinfo.url=file:///autonode/annotation.json - - -hostname=@/autonode/hostname - - # jostler archives the raw data from ndt-server and the uuid-annotator into - # JSONL "bundles". These bundles are uploaded to GCS using credentials from - # register. Jostler depends on the local schemas being up to date or backward - # compatible with upstream schemas. - jostler: - image: measurementlab/jostler:v1.1.4 - profiles: [ndt] - volumes: - - ./resultsdir:/resultsdir - - ./schemas:/schemas - - ./certs:/certs - - ./autonode:/autonode - network_mode: host - depends_on: - generate-schemas-ndt7: - condition: service_completed_successfully - generate-schemas-annotation2: - condition: service_completed_successfully - generate-schemas-traceroute: - condition: service_completed_successfully - register-node: - condition: service_healthy - restart: always - environment: - - GOOGLE_APPLICATION_CREDENTIALS=/autonode/service-account-autojoin.json - # NOTE: jostler should not restart on exit. - command: - - -mlab-node-name=@/autonode/hostname - # NOTE: the ndt7 schema must already exist in the target bucket. - - -gcs-bucket=archive-${PROJECT} - - -gcs-data-dir=autoload/v2 - - -local-data-dir=/resultsdir - - -organization=${ORGANIZATION} - - -experiment=ndt - - -datatype=ndt7 - - -datatype-schema-file=ndt7:/schemas/ndt7.json - - -datatype=annotation2 - - -datatype-schema-file=annotation2:/schemas/annotation2.json - - -datatype=scamper2 - - -datatype-schema-file=scamper2:/schemas/scamper2.json - - -datatype=hopannotation2 - - -datatype-schema-file=hopannotation2:/schemas/hopannotation2.json - - -bundle-size-max=20971520 - - -bundle-age-max=1h - - -missed-age=2h - - -missed-interval=5m - - -extensions=.json - - -upload-schema=false - - -verbose - - -prometheusx.listen-address=:9991 - - traceroute-caller: - image: measurementlab/traceroute-caller:v0.12.0 - profiles: [ndt] - network_mode: host - volumes: - - ./resultsdir:/resultsdir - - ./schemas:/schemas - - ./metadata:/metadata - depends_on: - generate-schemas-traceroute: - condition: service_completed_successfully - register-node: - condition: service_healthy - ndt-server: - condition: service_healthy - pull_policy: always - restart: always - configs: - - source: metadata-external-ipv6 - target: /metadata/external-ipv6 - - source: metadata-external-ip - target: /metadata/external-ip - command: - - -prometheusx.listen-address=:9994 - - -traceroute-output=/resultsdir/ndt/scamper2 - - -hopannotation-output=/resultsdir/ndt/hopannotation2 - - -tcpinfo.eventsocket=/schemas/events.sock - - -IPCacheTimeout=10m - - -IPCacheUpdatePeriod=1m - - -scamper.timeout=30m - - -scamper.waitprobe=15 - - -scamper.ptr=true - - -ipservice.sock=/schemas/uuid-annotator.sock - - -anonymize.ip=none - - -scamper.trace-type=regular - - -output.format=json - - node-exporter: - image: quay.io/prometheus/node-exporter:v1.9.0 - profiles: [ndt] - container_name: node-exporter - restart: always - network_mode: host - pid: host - volumes: - - /:/host:ro,rslave - command: - - --web.listen-address=:9995 - - --path.rootfs=/host - - --collector.disable-defaults - - --collector.cpu - - --collector.diskstats - - --collector.filesystem - - --collector.loadavg - - --collector.meminfo - - --collector.netdev - - --collector.stat - - --collector.time - - # Generate the schemas needed by jostler to assess that the data is backward - # compatible with upstream schemas and safe to upload. - generate-schemas-ndt7: - network_mode: host - image: measurementlab/ndt-server:v0.25.3 - volumes: - - ./schemas:/schemas - entrypoint: - - /generate-schemas - - -ndt7=/schemas/ndt7.json - - generate-schemas-annotation2: - network_mode: host - image: measurementlab/uuid-annotator:v0.5.8 - volumes: - - ./schemas:/schemas - entrypoint: - - /generate-schemas - - -ann2=/schemas/annotation2.json - - generate-schemas-traceroute: - network_mode: host - image: measurementlab/traceroute-caller:v0.12.0 - volumes: - - ./schemas:/schemas - entrypoint: - - /generate-schemas - - -scamper2=/schemas/scamper2.json - - -hopannotation2=/schemas/hopannotation2.json - - # Check that the tcp_bbr module is available on the host. This is required - # for ndt-server to use the BBR congestion control algorithm. - check-bbr: - image: bash:latest - command: ["sh", "-c", "lsmod | grep tcp_bbr"] - -configs: - locate-verify-key: - content: | - {"use":"sig","kty":"OKP","kid":"locate_20200409","crv":"Ed25519","alg":"EdDSA","x":"1tS1d-dd2B-VRBTWzOaq7zUngKDyV409K-o42LN2nx8"} - metadata-external-ip: - content: |- - ${IPV4} - metadata-external-ipv6: - content: |- - ${IPV6} diff --git a/env b/env deleted file mode 100644 index 63b325d..0000000 --- a/env +++ /dev/null @@ -1,93 +0,0 @@ -# This is an example .env file for the ndt-fullstack docker compose file. -# -# ### External directories ### -# -# These folders are mounted by the various containers in the fullstack NDT -# Docker compose. If they do not exist, they are created the first time the -# containers start, owned by the docker daemon's user (generally root). -# -# The only folder that MUST exist and contain the service account key file -# (see below) is ./certs. This file must also be readable by Docker's user. -# -# All other folders are created and managed by the containers in the Docker -# compose. -# -# ./html: Content served by ndt-server. Can be used to provide your own -# speedtest UI. -# -# ./certs: Contains the service account key file. This key file (provided by -# M-Lab) is expected to be placed at certs/service-account-autojoin.json. -# This file must be present for several components in the docker compose to -# work properly. -# -# ./resultsdir: Contains the test results waiting to be uploaded. -# -# ./schemas: Contains the JSON schemas for the test results. -# -# ./autocert: Contains the automatically-generated LetsEncrypt TLS certificates -# so that they can be reused across docker compose restarts. -# -# ./autonode: Contains the configuration files for the NDT server and all the -# sidecar containers, provided by the Autojoin API on node registration. -# -# ./metadata: Contains configuration files used by some services such as -# traceroute-caller to read external IP addresses in some environments. - -# ### Configuration variables ### -# Change the following settings according to your environment. - -# ORGANIZATION is the organization name assigned by M-Lab to your deployment. -ORGANIZATION= - -# API_KEY is the API key provided by M-Lab. -API_KEY= - -# IATA is the 3-letter, lowercase IATA code of the nearest international -# airport. You can use the following service to estimate it, providing your -# server's latitude and longitude: -# https://autojoin-dot-mlab-autojoin.appspot.com/autojoin/v0/lookup?lat=&lon= -IATA= - -# PROBABILITY is the probability that this node will be selected by M-Lab's -# load balancer. Tweak it according to how much traffic this node can handle. -# A probability of 1 (100%) means that this node will be equally as likely to -# be selected as any other M-Lab node in the same metro. Default is 0.1 (10%). -PROBABILITY=0.1 - -# INTERFACE_NAME is the name of the interface to apply INTERFACE_MAXRATE to. -INTERFACE_NAME= - -# UPLINK is the speed of the uplink the M-Lab machine is connected to. The value -# must be expressed as an integer followed by the letter "g" (e.g., "1g", "10g", -# "25g"), where "g" represents Gbps (gigabits per second). The minimum uplink -# M-lab requires is 1g, but 10g or greater is recommended. -UPLINK= - -# INTERFACE_MAXRATE is the rate (in bit/s) after which the NDT server will -# refuse new connections. Set it according to this node's uplink defined in -# UPLINK. -# Recommended values: -# - 1G: INTERFACE_MAXRATE=150000000 -# - 10G: INTERFACE_MAXRATE=7000000000 -INTERFACE_MAXRATE= - -# IPV4 is the external IPv4 address that this node will be reachable at. -# This is required. -IPV4= - -# IPV6 is the external IPv6 address that this node will be reachable at. -# If you want you node to be accessible over IPv6, this must be set. -IPV6= - -# TYPE is the type of machine that will be running the M-Lab software. The only -# valid values are "physical" or "virtual". This helps M-Lab understand the -# environment where the software is running. In other words, is the machine a -# bare-metal machine dedicated to M-Lab, or a VM? -TYPE= - -# ### Static configuration variables ### -# ### Do NOT change these. ### - -PROJECT=mlab-autojoin -LOCATE_URL=locate.measurementlab.net -AUTH_URL=auth.mlab-oti.measurementlab.net diff --git a/html/index.html b/html/index.html new file mode 100644 index 0000000..4dbf4f1 --- /dev/null +++ b/html/index.html @@ -0,0 +1,6 @@ + +M-Lab NDT server +

M-Lab NDT server

+

This is an M-Lab Network +Diagnostic Tool (NDT) measurement server. See the +autonode project for details.

diff --git a/scripts/gen-uuid-prefix b/scripts/gen-uuid-prefix new file mode 100755 index 0000000..a38a2d9 --- /dev/null +++ b/scripts/gen-uuid-prefix @@ -0,0 +1,9 @@ +#!/bin/sh +# Generate the ndt-server UUID prefix from the kernel boot time and hostname. +# Runs once per boot (btime only changes across reboots). Replaces the inline +# /bin/sh entrypoint the ndt-server container used under docker-compose. +set -eu + +prefix_file="/var/lib/mlab/schemas/uuid.prefix" +btime="$(awk '/^btime / {print $2}' /proc/stat)" +printf '%s_%s' "$(hostname)" "${btime}" > "${prefix_file}" diff --git a/scripts/wait-socket b/scripts/wait-socket new file mode 100755 index 0000000..8cdc217 --- /dev/null +++ b/scripts/wait-socket @@ -0,0 +1,19 @@ +#!/bin/sh +# Wait for one or more unix sockets to appear before starting. `After=` only +# guarantees the producing service's process has started, not that it has +# created its socket; this closes that race for the tcpinfo/ipservice sockets. +# All sockets are watched in the same loop, so the timeout is a single 60s +# budget rather than 60s per socket. +set -eu + +[ $# -ge 1 ] || { echo "usage: wait-socket SOCKET_PATH..." >&2; exit 2; } +for _ in $(seq 1 60); do + ok=1 + for sock in "$@"; do + [ -S "${sock}" ] || { ok=0; break; } + done + [ "${ok}" -eq 1 ] && exit 0 + sleep 1 +done +echo "mlab-node: socket(s) $* did not appear in time" >&2 +exit 1 diff --git a/scripts/write-metadata b/scripts/write-metadata new file mode 100755 index 0000000..13a682f --- /dev/null +++ b/scripts/write-metadata @@ -0,0 +1,10 @@ +#!/bin/sh +# Write the external IP metadata files consumed by traceroute-caller. Replaces +# the docker-compose "configs" that injected ${IPV4}/${IPV6} as files. +# IPV4/IPV6 come from the mlab-node EnvironmentFile. +set -eu + +: "${IPV4:=}" +: "${IPV6:=}" +printf '%s' "${IPV4}" > /run/mlab/metadata/external-ip +printf '%s' "${IPV6}" > /run/mlab/metadata/external-ipv6 diff --git a/systemd/mlab-node-bbr.service b/systemd/mlab-node-bbr.service new file mode 100644 index 0000000..bae61be --- /dev/null +++ b/systemd/mlab-node-bbr.service @@ -0,0 +1,13 @@ +[Unit] +Description=Ensure tcp_bbr kernel module is loaded for ndt-server +Documentation=https://github.com/m-lab/autonode +PartOf=mlab-node.target +After=systemd-modules-load.service +# Replaces the docker-compose check-bbr container. modprobe both loads the +# module (covering a fresh install before the next reboot) and fails loudly if +# the kernel cannot provide it. modules-load.d also loads it at boot. + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/sbin/modprobe tcp_bbr diff --git a/systemd/mlab-node-heartbeat.service b/systemd/mlab-node-heartbeat.service new file mode 100644 index 0000000..7844855 --- /dev/null +++ b/systemd/mlab-node-heartbeat.service @@ -0,0 +1,55 @@ +[Unit] +Description=M-Lab heartbeat (reports liveness to the Locate API) +Documentation=https://github.com/m-lab/autonode +PartOf=mlab-node.target +Requires=mlab-node-register-ready.service +After=mlab-node-register-ready.service +# Soft dependency on ndt-server (compose used service_started): order after it +# but do not tear heartbeat down if ndt-server restarts. +Wants=mlab-node-ndt-server.service +After=mlab-node-ndt-server.service +# Atomic lifecycle: keystone is register-node. +BindsTo=mlab-node-register.service + +[Service] +Type=exec +EnvironmentFile=/usr/lib/mlab/static.env +EnvironmentFile=/etc/mlab/mlab-node.env +EnvironmentFile=/etc/mlab/api-key.env +ExecStart=/usr/lib/mlab/heartbeat \ + -prometheusx.listen-address=:9993 \ + -experiment=ndt \ + -hostname=@/var/lib/mlab/node/hostname \ + -registration-url=file:///var/lib/mlab/node/registration.json \ + -heartbeat-url=wss://${LOCATE_URL}/v2/platform/heartbeat-jwt \ + -api-key=${API_KEY} \ + -token-exchange-url=https://${AUTH_URL}/v0/token/autojoin \ + -services=ndt/ndt7=ws:///ndt/v7/download,ws:///ndt/v7/upload,wss:///ndt/v7/download,wss:///ndt/v7/upload +Restart=always +RestartSec=5 + +# --- hardening --- +User=mlab-node +Group=mlab-node +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +ProtectProc=invisible +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes +PrivateTmp=yes +PrivateDevices=yes +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +LockPersonality=yes +MemoryDenyWriteExecute=yes +RemoveIPC=yes +CapabilityBoundingSet= +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +SystemCallFilter=@system-service +SystemCallErrorNumber=EPERM diff --git a/systemd/mlab-node-jostler.service b/systemd/mlab-node-jostler.service new file mode 100644 index 0000000..cf3c074 --- /dev/null +++ b/systemd/mlab-node-jostler.service @@ -0,0 +1,67 @@ +[Unit] +Description=M-Lab jostler (archives results to GCS) +Documentation=https://github.com/m-lab/autonode +PartOf=mlab-node.target +# Requires all schemas to be generated (service_completed_successfully) and a +# successful registration. +Requires=mlab-node-register-ready.service mlab-node-schemas-ndt7.service mlab-node-schemas-annotation2.service mlab-node-schemas-traceroute.service +After=mlab-node-register-ready.service mlab-node-schemas-ndt7.service mlab-node-schemas-annotation2.service mlab-node-schemas-traceroute.service +BindsTo=mlab-node-register.service + +[Service] +Type=exec +EnvironmentFile=/usr/lib/mlab/static.env +EnvironmentFile=/etc/mlab/mlab-node.env +Environment=GOOGLE_APPLICATION_CREDENTIALS=/var/lib/mlab/node/service-account-autojoin.json +ExecStart=/usr/lib/mlab/jostler \ + -mlab-node-name=@/var/lib/mlab/node/hostname \ + -gcs-bucket=archive-${PROJECT} \ + -gcs-data-dir=autoload/v2 \ + -local-data-dir=/var/lib/mlab/resultsdir \ + -organization=${ORGANIZATION} \ + -experiment=ndt \ + -datatype=ndt7 \ + -datatype-schema-file=ndt7:/var/lib/mlab/schemas/ndt7.json \ + -datatype=annotation2 \ + -datatype-schema-file=annotation2:/var/lib/mlab/schemas/annotation2.json \ + -datatype=scamper2 \ + -datatype-schema-file=scamper2:/var/lib/mlab/schemas/scamper2.json \ + -datatype=hopannotation2 \ + -datatype-schema-file=hopannotation2:/var/lib/mlab/schemas/hopannotation2.json \ + -bundle-size-max=20971520 \ + -bundle-age-max=1h \ + -missed-age=2h \ + -missed-interval=5m \ + -extensions=.json \ + -upload-schema=false \ + -verbose \ + -prometheusx.listen-address=:9991 +Restart=always +RestartSec=5 + +# --- hardening --- +User=mlab-node +Group=mlab-node +ReadWritePaths=/var/lib/mlab/resultsdir +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +ProtectProc=invisible +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes +PrivateTmp=yes +PrivateDevices=yes +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +LockPersonality=yes +MemoryDenyWriteExecute=yes +RemoveIPC=yes +CapabilityBoundingSet= +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +SystemCallFilter=@system-service +SystemCallErrorNumber=EPERM diff --git a/systemd/mlab-node-metadata.service b/systemd/mlab-node-metadata.service new file mode 100644 index 0000000..2a81ac3 --- /dev/null +++ b/systemd/mlab-node-metadata.service @@ -0,0 +1,24 @@ +[Unit] +Description=Write external IP metadata files for traceroute-caller +Documentation=https://github.com/m-lab/autonode +PartOf=mlab-node.target +# Replaces the docker-compose "configs" that injected ${IPV4}/${IPV6} as the +# /metadata/external-ip[v6] files. Regenerated each boot (the files live on the +# /run tmpfs). + +[Service] +Type=oneshot +RemainAfterExit=yes +EnvironmentFile=/etc/mlab/mlab-node.env +ExecStart=/usr/lib/mlab/write-metadata + +# --- hardening --- +User=mlab-node +Group=mlab-node +ReadWritePaths=/run/mlab/metadata +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +RestrictAddressFamilies=AF_UNIX +CapabilityBoundingSet= diff --git a/systemd/mlab-node-ndt-server.service b/systemd/mlab-node-ndt-server.service new file mode 100644 index 0000000..8ce5be6 --- /dev/null +++ b/systemd/mlab-node-ndt-server.service @@ -0,0 +1,90 @@ +[Unit] +Description=M-Lab ndt-server (NDT7 measurement service) +Documentation=https://github.com/m-lab/autonode +PartOf=mlab-node.target +# Start ordering / gating (replaces docker-compose depends_on): +# register-ready -> registration succeeded (service_healthy) +# schemas-ndt7 -> ndt7 schema generated (service_completed_successfully) +# uuidprefix/bbr -> prerequisites generated/loaded +Requires=mlab-node-register-ready.service mlab-node-schemas-ndt7.service mlab-node-uuidprefix.service mlab-node-bbr.service +After=mlab-node-register-ready.service mlab-node-schemas-ndt7.service mlab-node-uuidprefix.service mlab-node-bbr.service +# Atomic lifecycle: ndt-server cannot run without an active registration. +BindsTo=mlab-node-register.service + +[Service] +Type=exec +EnvironmentFile=/etc/mlab/mlab-node.env +# Clear any stale tcpinfo socket left by a previous instance before binding. +ExecStartPre=/bin/rm -f /run/mlab/sockets/events.sock +ExecStart=/usr/lib/mlab/ndt-server \ + -uuid-prefix-file=/var/lib/mlab/schemas/uuid.prefix \ + -datadir=/var/lib/mlab/resultsdir/ndt \ + -htmldir=/usr/share/mlab/html \ + -ndt7_addr=:443 \ + -ndt7_addr_cleartext=:80 \ + -compress-results=false \ + -txcontroller.device=${INTERFACE_NAME} \ + -txcontroller.max-rate=${INTERFACE_MAXRATE} \ + -prometheusx.listen-address=:9990 \ + -label=type=${TYPE} \ + -label=deployment=byos \ + -label=managed=none \ + -label=loadbalanced=false \ + -label=org=${ORGANIZATION} \ + -label=external-ip=${IPV4} \ + -label=external-ipv6=${IPV6} \ + -ndt5_addr=127.0.0.1:3002 \ + -ndt5_ws_addr=127.0.0.1:3001 \ + -tcpinfo.eventsocket=/run/mlab/sockets/events.sock \ + -autocert.enabled=true \ + -autocert.dir=/var/lib/mlab/autocert \ + -autocert.hostname=@/var/lib/mlab/node/hostname \ + -token.machine=@/var/lib/mlab/node/hostname \ + -token.verify-key=/etc/mlab/locate/verify.pub \ + -ndt7.token.required=true +Restart=always +RestartSec=5 + +# Graceful (lame-duck) stop. ndt-server's signal protocol is two SIGTERMs: +# the 1st flips /health (127.0.0.1:8000) to 500 — heartbeat notices within +# 10s and reports score 0, so Locate stops directing new clients here — and +# the 2nd exits the server. ExecStop sends the 1st and waits out the drain +# window; when it returns, systemd delivers its regular KillSignal (SIGTERM) +# as the 2nd, and the server exits cleanly with in-flight tests finished. +# The window mirrors the k8s platform's 100s termination grace period (10s +# heartbeat notice + 30s Locate propagation + 60s max test length). Do not +# send the 2nd SIGTERM from the script: ndt-server closes its signal channel +# after the 2nd, so a 3rd could panic the exiting process. +ExecStop=/bin/sh -c 'kill -s TERM "$MAINPID"; sleep 100' +# Bounds each stop phase (ExecStop above, then the SIGTERM kill phase) +# separately; must exceed the 100s drain or systemd would SIGKILL mid-drain. +TimeoutStopSec=130 + +# --- hardening --- +User=mlab-node +Group=mlab-node +# Bind the privileged ports 80/443 without running as root. +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +ReadWritePaths=/var/lib/mlab/resultsdir /var/lib/mlab/autocert /run/mlab/sockets +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +ProtectProc=invisible +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes +PrivateTmp=yes +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +LockPersonality=yes +MemoryDenyWriteExecute=yes +RemoveIPC=yes +# AF_NETLINK is required by -txcontroller to read the interface transmit rate. +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK +SystemCallFilter=@system-service +SystemCallErrorNumber=EPERM diff --git a/systemd/mlab-node-node-exporter.service b/systemd/mlab-node-node-exporter.service new file mode 100644 index 0000000..dba1253 --- /dev/null +++ b/systemd/mlab-node-node-exporter.service @@ -0,0 +1,51 @@ +[Unit] +Description=Prometheus node exporter for the M-Lab node +Documentation=https://github.com/m-lab/autonode +PartOf=mlab-node.target +After=network-online.target +Wants=network-online.target +# Part of the atomic stack: brought down with the keystone register-node. +BindsTo=mlab-node-register.service + +[Service] +Type=exec +# Running natively on the host, so the default rootfs (/) is correct; the +# --path.rootfs=/host indirection used under Docker is dropped. +ExecStart=/usr/lib/mlab/node-exporter \ + --web.listen-address=:9995 \ + --collector.disable-defaults \ + --collector.cpu \ + --collector.diskstats \ + --collector.filesystem \ + --collector.loadavg \ + --collector.meminfo \ + --collector.netdev \ + --collector.stat \ + --collector.time +Restart=always +RestartSec=5 + +# --- hardening --- +# node-exporter shares nothing with the rest of the stack and needs no state, so +# it runs under an ephemeral DynamicUser. It only reads world-readable /proc and +# /sys, so it needs no capabilities and no write access. ProtectProc= is left at +# its default: the collectors read system-wide /proc files. +DynamicUser=yes +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +PrivateTmp=yes +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +LockPersonality=yes +MemoryDenyWriteExecute=yes +CapabilityBoundingSet= +RestrictAddressFamilies=AF_INET AF_INET6 +SystemCallFilter=@system-service +SystemCallErrorNumber=EPERM diff --git a/systemd/mlab-node-register-ready.service b/systemd/mlab-node-register-ready.service new file mode 100644 index 0000000..2719ee8 --- /dev/null +++ b/systemd/mlab-node-register-ready.service @@ -0,0 +1,28 @@ +[Unit] +Description=Wait for M-Lab node registration to be ready +Documentation=https://github.com/m-lab/autonode +PartOf=mlab-node.target +After=mlab-node-register.service +BindsTo=mlab-node-register.service +# This oneshot gates every dependent component on a successful registration +# (the register-node :8001/ready endpoint), guaranteeing the hostname, +# registration.json and service-account files exist before anything reads them. + +[Service] +Type=oneshot +RemainAfterExit=yes +# Poll until registration reports ready: retry connection-refused (register +# still starting) and HTTP errors (not yet registered) for up to ~120s. +ExecStart=/usr/bin/curl -fsS -o /dev/null \ + --retry 60 --retry-delay 2 --retry-connrefused --retry-all-errors \ + http://localhost:8001/ready + +# --- hardening --- +User=mlab-node +Group=mlab-node +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +CapabilityBoundingSet= diff --git a/systemd/mlab-node-register.service b/systemd/mlab-node-register.service new file mode 100644 index 0000000..dd11fd5 --- /dev/null +++ b/systemd/mlab-node-register.service @@ -0,0 +1,60 @@ +[Unit] +Description=M-Lab node registration (Autojoin API) +Documentation=https://github.com/m-lab/autonode +PartOf=mlab-node.target +After=network-online.target +Wants=network-online.target +# register-node is the keystone of the stack: every other component BindsTo= +# this unit, so if registration goes down the whole stack is brought down with +# it (it cannot run without a valid registration). A genuine crash-loop is +# bounded by the start limit below, at which point the stack stays down. +StartLimitIntervalSec=600 +StartLimitBurst=5 + +[Service] +Type=exec +EnvironmentFile=/usr/lib/mlab/static.env +EnvironmentFile=/etc/mlab/mlab-node.env +EnvironmentFile=/etc/mlab/api-key.env +ExecStart=/usr/lib/mlab/autojoin-register \ + -endpoint=https://autojoin-dot-${PROJECT}.appspot.com/autojoin/v0/node/register \ + -key=${API_KEY} \ + -service=ndt \ + -iata=${IATA} \ + -output=/var/lib/mlab/node \ + -healthcheck-addr=:8001 \ + -ports=9990,9991,9992,9993,9994,9995 \ + -probability=${PROBABILITY} \ + -ipv4=${IPV4} \ + -ipv6=${IPV6} \ + -type=${TYPE} \ + -uplink=${UPLINK} +Restart=always +RestartSec=10 + +# --- hardening --- +User=mlab-node +Group=mlab-node +ReadWritePaths=/var/lib/mlab/node +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +ProtectProc=invisible +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes +PrivateTmp=yes +PrivateDevices=yes +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +LockPersonality=yes +MemoryDenyWriteExecute=yes +RemoveIPC=yes +CapabilityBoundingSet= +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +SystemCallFilter=@system-service +SystemCallErrorNumber=EPERM diff --git a/systemd/mlab-node-schemas-annotation2.service b/systemd/mlab-node-schemas-annotation2.service new file mode 100644 index 0000000..9848bc0 --- /dev/null +++ b/systemd/mlab-node-schemas-annotation2.service @@ -0,0 +1,21 @@ +[Unit] +Description=Generate annotation2 JSON schema +Documentation=https://github.com/m-lab/autonode +PartOf=mlab-node.target +# Replaces the generate-schemas-annotation2 init container. + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/lib/mlab/generate-schemas-annotation2 -ann2=/var/lib/mlab/schemas/annotation2.json + +# --- hardening --- +User=mlab-node +Group=mlab-node +ReadWritePaths=/var/lib/mlab/schemas +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +RestrictAddressFamilies=AF_UNIX +CapabilityBoundingSet= diff --git a/systemd/mlab-node-schemas-ndt7.service b/systemd/mlab-node-schemas-ndt7.service new file mode 100644 index 0000000..ffbbd3e --- /dev/null +++ b/systemd/mlab-node-schemas-ndt7.service @@ -0,0 +1,22 @@ +[Unit] +Description=Generate ndt7 JSON schema +Documentation=https://github.com/m-lab/autonode +PartOf=mlab-node.target +# Replaces the generate-schemas-ndt7 init container. jostler validates uploads +# against these schemas; ndt-server also requires the ndt7 schema. + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/lib/mlab/generate-schemas-ndt7 -ndt7=/var/lib/mlab/schemas/ndt7.json + +# --- hardening --- +User=mlab-node +Group=mlab-node +ReadWritePaths=/var/lib/mlab/schemas +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +RestrictAddressFamilies=AF_UNIX +CapabilityBoundingSet= diff --git a/systemd/mlab-node-schemas-traceroute.service b/systemd/mlab-node-schemas-traceroute.service new file mode 100644 index 0000000..9e27f09 --- /dev/null +++ b/systemd/mlab-node-schemas-traceroute.service @@ -0,0 +1,23 @@ +[Unit] +Description=Generate scamper2 and hopannotation2 JSON schemas +Documentation=https://github.com/m-lab/autonode +PartOf=mlab-node.target +# Replaces the generate-schemas-traceroute init container. + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/lib/mlab/generate-schemas-traceroute \ + -scamper2=/var/lib/mlab/schemas/scamper2.json \ + -hopannotation2=/var/lib/mlab/schemas/hopannotation2.json + +# --- hardening --- +User=mlab-node +Group=mlab-node +ReadWritePaths=/var/lib/mlab/schemas +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +RestrictAddressFamilies=AF_UNIX +CapabilityBoundingSet= diff --git a/systemd/mlab-node-traceroute-caller.service b/systemd/mlab-node-traceroute-caller.service new file mode 100644 index 0000000..05aa5ba --- /dev/null +++ b/systemd/mlab-node-traceroute-caller.service @@ -0,0 +1,63 @@ +[Unit] +Description=M-Lab traceroute-caller (runs scamper traceroutes) +Documentation=https://github.com/m-lab/autonode +PartOf=mlab-node.target +Requires=mlab-node-register-ready.service mlab-node-schemas-traceroute.service mlab-node-metadata.service +After=mlab-node-register-ready.service mlab-node-schemas-traceroute.service mlab-node-metadata.service +# Compose required ndt-server to be healthy (traceroute-caller connects to its +# events.sock); order after it and wait for the socket in ExecStartPre. +Wants=mlab-node-ndt-server.service +After=mlab-node-ndt-server.service +BindsTo=mlab-node-register.service + +[Service] +Type=exec +EnvironmentFile=/etc/mlab/mlab-node.env +ExecStartPre=/usr/lib/mlab/wait-socket /run/mlab/sockets/events.sock /run/mlab/sockets/uuid-annotator.sock +ExecStart=/usr/lib/mlab/traceroute-caller \ + -scamper.bin=/usr/lib/mlab/scamper \ + -prometheusx.listen-address=:9994 \ + -traceroute-output=/var/lib/mlab/resultsdir/ndt/scamper2 \ + -hopannotation-output=/var/lib/mlab/resultsdir/ndt/hopannotation2 \ + -tcpinfo.eventsocket=/run/mlab/sockets/events.sock \ + -IPCacheTimeout=10m \ + -IPCacheUpdatePeriod=1m \ + -scamper.timeout=30m \ + -scamper.waitprobe=15 \ + -scamper.ptr=true \ + -ipservice.sock=/run/mlab/sockets/uuid-annotator.sock \ + -anonymize.ip=none \ + -scamper.trace-type=regular \ + -output.format=json +Restart=always +RestartSec=5 + +# --- hardening --- +User=mlab-node +Group=mlab-node +# scamper needs raw sockets to send/receive probes; the child inherits the +# ambient capability from traceroute-caller. +AmbientCapabilities=CAP_NET_RAW +CapabilityBoundingSet=CAP_NET_RAW +ReadWritePaths=/var/lib/mlab/resultsdir +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +ProtectProc=invisible +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes +PrivateTmp=yes +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +LockPersonality=yes +# Relaxed for the bundled scamper C binary. +MemoryDenyWriteExecute=no +RemoveIPC=yes +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET +SystemCallFilter=@system-service +SystemCallErrorNumber=EPERM diff --git a/systemd/mlab-node-uuid-annotator.service b/systemd/mlab-node-uuid-annotator.service new file mode 100644 index 0000000..90321fd --- /dev/null +++ b/systemd/mlab-node-uuid-annotator.service @@ -0,0 +1,62 @@ +[Unit] +Description=M-Lab uuid-annotator (network/geo annotations) +Documentation=https://github.com/m-lab/autonode +PartOf=mlab-node.target +Requires=mlab-node-register-ready.service +After=mlab-node-register-ready.service +# Soft dependency on ndt-server (compose used service_started); the annotator +# connects to ndt-server's tcpinfo events.sock. +Wants=mlab-node-ndt-server.service +After=mlab-node-ndt-server.service +BindsTo=mlab-node-register.service + +[Service] +Type=exec +EnvironmentFile=/usr/lib/mlab/static.env +EnvironmentFile=/etc/mlab/mlab-node.env +Environment=GOOGLE_APPLICATION_CREDENTIALS=/var/lib/mlab/node/service-account-autojoin.json +# Wait for ndt-server's events socket, and clear any stale annotator socket. +ExecStartPre=/usr/lib/mlab/wait-socket /run/mlab/sockets/events.sock +ExecStartPre=/bin/rm -f /run/mlab/sockets/uuid-annotator.sock +ExecStart=/usr/lib/mlab/uuid-annotator \ + -prometheusx.listen-address=:9992 \ + -datadir=/var/lib/mlab/resultsdir/ndt/annotation2 \ + -tcpinfo.eventsocket=/run/mlab/sockets/events.sock \ + -ipservice.sock=/run/mlab/sockets/uuid-annotator.sock \ + -maxmind.url=gs://downloader-${PROJECT}/Maxmind/current/GeoLite2-City.tar.gz \ + -routeview-v4.url=gs://downloader-${PROJECT}/RouteViewIPv4/current/routeview.pfx2as.gz \ + -routeview-v6.url=gs://downloader-${PROJECT}/RouteViewIPv6/current/routeview.pfx2as.gz \ + -siteinfo.url=file:///var/lib/mlab/node/annotation.json \ + -asname.url=file:///usr/share/mlab/asnames.ipinfo.csv \ + -hostname=@/var/lib/mlab/node/hostname +Restart=always +RestartSec=5 + +# --- hardening --- +User=mlab-node +Group=mlab-node +ReadWritePaths=/var/lib/mlab/resultsdir /run/mlab/sockets +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +ProtectProc=invisible +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes +PrivateTmp=yes +PrivateDevices=yes +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +LockPersonality=yes +MemoryDenyWriteExecute=yes +RemoveIPC=yes +CapabilityBoundingSet= +# AF_NETLINK is required to enumerate the host's local addresses (the annotator +# reads them via netlink to annotate the server side of each measurement). +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK +SystemCallFilter=@system-service +SystemCallErrorNumber=EPERM diff --git a/systemd/mlab-node-uuidprefix.service b/systemd/mlab-node-uuidprefix.service new file mode 100644 index 0000000..92be4f1 --- /dev/null +++ b/systemd/mlab-node-uuidprefix.service @@ -0,0 +1,22 @@ +[Unit] +Description=Generate ndt-server UUID prefix from boot time +Documentation=https://github.com/m-lab/autonode +PartOf=mlab-node.target +# Runs once per boot; btime (from /proc/stat) only changes across reboots. +# Replaces the inline shell entrypoint the ndt-server container used. + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/lib/mlab/gen-uuid-prefix + +# --- hardening --- +User=mlab-node +Group=mlab-node +ReadWritePaths=/var/lib/mlab/schemas +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +RestrictAddressFamilies=AF_UNIX +CapabilityBoundingSet= diff --git a/systemd/mlab-node.target b/systemd/mlab-node.target new file mode 100644 index 0000000..fe1da6a --- /dev/null +++ b/systemd/mlab-node.target @@ -0,0 +1,16 @@ +[Unit] +Description=M-Lab node (autonode) measurement stack +Documentation=https://github.com/m-lab/autonode +# The target is the single handle for the whole stack, replacing +# `docker compose up/down`. Starting it pulls in every component; stopping it +# stops them all (each component is PartOf= this target). +Wants=mlab-node-register.service +Wants=mlab-node-ndt-server.service +Wants=mlab-node-heartbeat.service +Wants=mlab-node-uuid-annotator.service +Wants=mlab-node-jostler.service +Wants=mlab-node-traceroute-caller.service +Wants=mlab-node-node-exporter.service + +[Install] +WantedBy=multi-user.target