From c8b0f8515b486e322e29d4d85cde7147639b6a09 Mon Sep 17 00:00:00 2001 From: Gregory Jefferis Date: Tue, 25 Aug 2026 13:55:38 +0100 Subject: [PATCH 1/2] ci: provision Python via simple_python(); sharpen module-skip helper Provision CI Python through nat.python's own simple_python() engine (pip pandas into the managed r-reticulate env) instead of reticulate::py_install(), which pulled a conda-forge pandas 3.0.5 that would not import against numpy 2.5.2 -- silently skipping every pandas2df() test and dropping coverage. The test helper now distinguishes an absent module (skip) from one that is installed but fails to import (error, surfacing the Python error) so a broken environment can no longer masquerade as a benign skip. --- .github/workflows/R-CMD-check.yaml | 24 +++++++++------ .github/workflows/test-coverage.yaml | 20 ++++++------ tests/testthat/helper-nat.python.R | 46 +++++++++++++++++++++++----- 3 files changed, 64 insertions(+), 26 deletions(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 38a3c7c..d745606 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -1,12 +1,12 @@ # Single-platform (Ubuntu) R CMD check, including the testthat suite, across a # small matrix of Python versions. # -# Python is provided the way real users get it -- through reticulate's managed -# miniconda -- rather than a bare system Python, so CI exercises the same kind -# of interpreter nat.python is designed to manage. nat.python itself is installed -# and used to introspect the environment before the check runs. (Phase 2 will -# provision Python through nat.python's own environment engine; until then we -# call reticulate directly.) +# Python is provided the way real users get it -- through nat.python's own +# environment engine (simple_python()), which installs miniconda, the +# r-reticulate env and pandas (via pip). This exercises the exact provisioning +# path nat.python ships, rather than a hand-rolled reticulate::py_install() call +# (whose conda-forge pandas could be an interpreter nat.python never gives a +# user). # # The matrix spans the Python range we care about on one platform: # * reticulate's own default miniconda Python (what most users get); @@ -53,19 +53,23 @@ jobs: extra-packages: any::rcmdcheck needs: check - - name: Provision Python (reticulate miniconda) and install nat.python + - name: Install nat.python and provision Python via simple_python() run: | v <- Sys.getenv("NATPY_PYTHON_VERSION") if (nzchar(v)) { + # simple_python() -> install_miniconda() honours this, so it pins the + # Python version of the managed r-reticulate environment. Sys.setenv(RETICULATE_MINICONDA_PYTHON_VERSION = v) - message("Pinning reticulate miniconda Python to ", v) + message("Pinning miniconda Python to ", v) } else { message("Using reticulate's default miniconda Python") } - reticulate::install_miniconda() - reticulate::py_install(c("numpy", "pandas")) pak::local_install() library(nat.python) + # The end-user path: provision miniconda + r-reticulate + pandas (via + # pip, pulling a mutually-compatible numpy). "minimal" is nat.python's + # own baseline, all pandas2df() needs. + simple_python("minimal") print(py_module_info(c("numpy", "pandas"))) py <- reticulate::py_config()$python cat(sprintf("RETICULATE_PYTHON=%s\n", py), diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index 09cb781..6dced39 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -1,7 +1,7 @@ # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples # Test coverage via covr, uploaded to Codecov. Python is provided the same way -# R-CMD-check.yaml does -- through reticulate's managed miniconda -- so coverage -# exercises the same kind of interpreter nat.python is designed to manage. +# R-CMD-check.yaml does -- through nat.python's own simple_python() engine -- so +# coverage exercises the exact provisioning path nat.python ships. on: push: branches: [main, master] @@ -30,16 +30,18 @@ jobs: extra-packages: any::covr, any::xml2 needs: coverage - - name: Provision Python (reticulate miniconda) and install nat.python - # Mirrors R-CMD-check.yaml's default leg: reticulate's own miniconda, - # numpy + pandas installed, then pin RETICULATE_PYTHON to that exact - # interpreter for the coverage run. + - name: Install nat.python and provision Python via simple_python() + # Mirrors R-CMD-check.yaml's default leg, but provisions through + # nat.python's own simple_python(): miniconda + r-reticulate + pandas + # (via pip). pyarrow is added so the use_arrow = TRUE conversion path is + # exercised. RETICULATE_PYTHON is then pinned to that interpreter for the + # coverage run. shell: Rscript {0} run: | - reticulate::install_miniconda() - # pyarrow lets the use_arrow = TRUE conversion path be exercised - reticulate::py_install(c("numpy", "pandas", "pyarrow")) pak::local_install() + library(nat.python) + simple_python("minimal", pkgs = "pyarrow") + print(py_module_info(c("numpy", "pandas", "pyarrow"))) cat(sprintf("RETICULATE_PYTHON=%s\n", reticulate::py_config()$python), file = Sys.getenv("GITHUB_ENV"), append = TRUE) diff --git a/tests/testthat/helper-nat.python.R b/tests/testthat/helper-nat.python.R index 97d311b..75c3b2e 100644 --- a/tests/testthat/helper-nat.python.R +++ b/tests/testthat/helper-nat.python.R @@ -1,11 +1,43 @@ -# Can we import a given Python module? Tests that need Python are skipped when -# it (or the module) is unavailable, so the suite still runs with no Python. -py_module_ok <- function(module) { - requireNamespace("reticulate", quietly = TRUE) && - !inherits(try(reticulate::import(module), silent = TRUE), "try-error") +# Classify a Python module's availability in the active environment. We keep the +# three cases apart on purpose (see skip_if_no_module): +# * "ok" -- importable, run the test; +# * "absent" -- no Python, or the module simply isn't installed -> skip; +# * "broken" -- the module *is* installed (per distribution metadata) but +# importing it fails -> a broken environment, not absence. +# The old helper collapsed "broken" into "absent", so an installed-but- +# unimportable module (e.g. a conda pandas built against an incompatible numpy) +# silently skipped and quietly dropped coverage instead of being noticed. +py_module_status <- function(module) { + if (!requireNamespace("reticulate", quietly = TRUE)) + return(list(state = "absent", error = NA_character_)) + if (!isTRUE(try(reticulate::py_available(initialize = TRUE), silent = TRUE))) + return(list(state = "absent", error = NA_character_)) + + # Try importing first: a clean import is the fast path, and it also covers + # stdlib/namespace modules that carry no distribution metadata (e.g. datetime). + imported <- try(reticulate::import(module), silent = TRUE) + if (!inherits(imported, "try-error")) + return(list(state = "ok", error = NA_character_)) + + err <- conditionMessage(attr(imported, "condition")) + + # Import failed. Is the module nonetheless installed, per distribution + # metadata (which does not import it)? If so, this is a broken environment. + installed <- tryCatch(isTRUE(nat.python::py_module_info(module)$available[1]), + error = function(e) FALSE) + + list(state = if (installed) "broken" else "absent", error = err) } +# Skip a test when a Python module is unavailable, but fail loudly when it is +# installed yet unimportable -- nat.python's job is to provision a working +# Python, so a broken module must never masquerade as a benign skip. skip_if_no_module <- function(module) { - testthat::skip_if_not(py_module_ok(module), - paste0("Python module '", module, "' not available")) + st <- py_module_status(module) + if (identical(st$state, "ok")) + return(invisible(TRUE)) + if (identical(st$state, "broken")) + stop(sprintf("Python module '%s' is installed but failed to import: %s", + module, st$error), call. = FALSE) + testthat::skip(paste0("Python module '", module, "' not available")) } From f226dda01442b570c1c420d7406cfc8af7a1cace Mon Sep 17 00:00:00 2001 From: Gregory Jefferis Date: Tue, 25 Aug 2026 14:01:45 +0100 Subject: [PATCH 2/2] env: pin simple_python baseline to pandas<3 pandas 3.0 defaults string columns to Arrow-backed dtype when pyarrow is present; pandas2df() does not yet convert ArrowStringArray back to R, so string columns returned as raw python objects and the fast-path tests failed. Pin the baseline until pandas2df() gains Arrow-string support. --- NEWS.md | 9 +++++++++ R/env.R | 9 +++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 0fc000c..40e3ec9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,12 @@ +# nat.python 0.2.0.9000 (development version) + +* `simple_python()` now pins the baseline install to `pandas < 3`. pandas 3.0 + makes Arrow-backed strings the default dtype, which `pandas2df()` does not yet + convert back to R; the pin keeps the provisioned environment functional until + that support lands (#6). +* CI now provisions Python through `simple_python()` itself (the end-user path), + rather than a bespoke `reticulate::py_install()` call. + # nat.python 0.2.0 First tagged release. A small shared layer of Python interoperability and diff --git a/R/env.R b/R/env.R index cc69496..8b57ce4 100644 --- a/R/env.R +++ b/R/env.R @@ -76,8 +76,13 @@ simple_python <- function(pyinstall = c("basic", "full", "extra", "minimal", if (pyinstall %in% c("minimal", "basic", "full", "extra")) { # nat.python's own baseline: pandas2df() needs pandas, and numpy rides in # with it. Every richer bundle builds on top of this. - cli::cli_inform("Installing pandas (brings numpy)") - ourpip("pandas") + # + # Pinned to pandas < 3 for now: pandas 3.0 makes Arrow-backed strings the + # default dtype (PDEP-14), which pandas2df() does not yet convert back to R + # (string columns return as raw ArrowStringArray objects). Lift this once + # pandas2df() handles the Arrow string dtype. + cli::cli_inform("Installing pandas (<3 for now; brings numpy)") + ourpip("pandas<3") } if (pyinstall %in% c("basic", "full", "extra")) { cli::cli_inform("Installing cloudvolume")