diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 38a3c7c..d745606 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -1,12 +1,12 @@ # Single-platform (Ubuntu) R CMD check, including the testthat suite, across a # small matrix of Python versions. # -# Python is provided the way real users get it -- through reticulate's managed -# miniconda -- rather than a bare system Python, so CI exercises the same kind -# of interpreter nat.python is designed to manage. nat.python itself is installed -# and used to introspect the environment before the check runs. (Phase 2 will -# provision Python through nat.python's own environment engine; until then we -# call reticulate directly.) +# Python is provided the way real users get it -- through nat.python's own +# environment engine (simple_python()), which installs miniconda, the +# r-reticulate env and pandas (via pip). This exercises the exact provisioning +# path nat.python ships, rather than a hand-rolled reticulate::py_install() call +# (whose conda-forge pandas could be an interpreter nat.python never gives a +# user). # # The matrix spans the Python range we care about on one platform: # * reticulate's own default miniconda Python (what most users get); @@ -53,19 +53,23 @@ jobs: extra-packages: any::rcmdcheck needs: check - - name: Provision Python (reticulate miniconda) and install nat.python + - name: Install nat.python and provision Python via simple_python() run: | v <- Sys.getenv("NATPY_PYTHON_VERSION") if (nzchar(v)) { + # simple_python() -> install_miniconda() honours this, so it pins the + # Python version of the managed r-reticulate environment. Sys.setenv(RETICULATE_MINICONDA_PYTHON_VERSION = v) - message("Pinning reticulate miniconda Python to ", v) + message("Pinning miniconda Python to ", v) } else { message("Using reticulate's default miniconda Python") } - reticulate::install_miniconda() - reticulate::py_install(c("numpy", "pandas")) pak::local_install() library(nat.python) + # The end-user path: provision miniconda + r-reticulate + pandas (via + # pip, pulling a mutually-compatible numpy). "minimal" is nat.python's + # own baseline, all pandas2df() needs. + simple_python("minimal") print(py_module_info(c("numpy", "pandas"))) py <- reticulate::py_config()$python cat(sprintf("RETICULATE_PYTHON=%s\n", py), diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index 09cb781..6dced39 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -1,7 +1,7 @@ # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples # Test coverage via covr, uploaded to Codecov. Python is provided the same way -# R-CMD-check.yaml does -- through reticulate's managed miniconda -- so coverage -# exercises the same kind of interpreter nat.python is designed to manage. +# R-CMD-check.yaml does -- through nat.python's own simple_python() engine -- so +# coverage exercises the exact provisioning path nat.python ships. on: push: branches: [main, master] @@ -30,16 +30,18 @@ jobs: extra-packages: any::covr, any::xml2 needs: coverage - - name: Provision Python (reticulate miniconda) and install nat.python - # Mirrors R-CMD-check.yaml's default leg: reticulate's own miniconda, - # numpy + pandas installed, then pin RETICULATE_PYTHON to that exact - # interpreter for the coverage run. + - name: Install nat.python and provision Python via simple_python() + # Mirrors R-CMD-check.yaml's default leg, but provisions through + # nat.python's own simple_python(): miniconda + r-reticulate + pandas + # (via pip). pyarrow is added so the use_arrow = TRUE conversion path is + # exercised. RETICULATE_PYTHON is then pinned to that interpreter for the + # coverage run. shell: Rscript {0} run: | - reticulate::install_miniconda() - # pyarrow lets the use_arrow = TRUE conversion path be exercised - reticulate::py_install(c("numpy", "pandas", "pyarrow")) pak::local_install() + library(nat.python) + simple_python("minimal", pkgs = "pyarrow") + print(py_module_info(c("numpy", "pandas", "pyarrow"))) cat(sprintf("RETICULATE_PYTHON=%s\n", reticulate::py_config()$python), file = Sys.getenv("GITHUB_ENV"), append = TRUE) diff --git a/NEWS.md b/NEWS.md index 0fc000c..40e3ec9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,12 @@ +# nat.python 0.2.0.9000 (development version) + +* `simple_python()` now pins the baseline install to `pandas < 3`. pandas 3.0 + makes Arrow-backed strings the default dtype, which `pandas2df()` does not yet + convert back to R; the pin keeps the provisioned environment functional until + that support lands (#6). +* CI now provisions Python through `simple_python()` itself (the end-user path), + rather than a bespoke `reticulate::py_install()` call. + # nat.python 0.2.0 First tagged release. A small shared layer of Python interoperability and diff --git a/R/env.R b/R/env.R index cc69496..8b57ce4 100644 --- a/R/env.R +++ b/R/env.R @@ -76,8 +76,13 @@ simple_python <- function(pyinstall = c("basic", "full", "extra", "minimal", if (pyinstall %in% c("minimal", "basic", "full", "extra")) { # nat.python's own baseline: pandas2df() needs pandas, and numpy rides in # with it. Every richer bundle builds on top of this. - cli::cli_inform("Installing pandas (brings numpy)") - ourpip("pandas") + # + # Pinned to pandas < 3 for now: pandas 3.0 makes Arrow-backed strings the + # default dtype (PDEP-14), which pandas2df() does not yet convert back to R + # (string columns return as raw ArrowStringArray objects). Lift this once + # pandas2df() handles the Arrow string dtype. + cli::cli_inform("Installing pandas (<3 for now; brings numpy)") + ourpip("pandas<3") } if (pyinstall %in% c("basic", "full", "extra")) { cli::cli_inform("Installing cloudvolume") diff --git a/tests/testthat/helper-nat.python.R b/tests/testthat/helper-nat.python.R index 97d311b..75c3b2e 100644 --- a/tests/testthat/helper-nat.python.R +++ b/tests/testthat/helper-nat.python.R @@ -1,11 +1,43 @@ -# Can we import a given Python module? Tests that need Python are skipped when -# it (or the module) is unavailable, so the suite still runs with no Python. -py_module_ok <- function(module) { - requireNamespace("reticulate", quietly = TRUE) && - !inherits(try(reticulate::import(module), silent = TRUE), "try-error") +# Classify a Python module's availability in the active environment. We keep the +# three cases apart on purpose (see skip_if_no_module): +# * "ok" -- importable, run the test; +# * "absent" -- no Python, or the module simply isn't installed -> skip; +# * "broken" -- the module *is* installed (per distribution metadata) but +# importing it fails -> a broken environment, not absence. +# The old helper collapsed "broken" into "absent", so an installed-but- +# unimportable module (e.g. a conda pandas built against an incompatible numpy) +# silently skipped and quietly dropped coverage instead of being noticed. +py_module_status <- function(module) { + if (!requireNamespace("reticulate", quietly = TRUE)) + return(list(state = "absent", error = NA_character_)) + if (!isTRUE(try(reticulate::py_available(initialize = TRUE), silent = TRUE))) + return(list(state = "absent", error = NA_character_)) + + # Try importing first: a clean import is the fast path, and it also covers + # stdlib/namespace modules that carry no distribution metadata (e.g. datetime). + imported <- try(reticulate::import(module), silent = TRUE) + if (!inherits(imported, "try-error")) + return(list(state = "ok", error = NA_character_)) + + err <- conditionMessage(attr(imported, "condition")) + + # Import failed. Is the module nonetheless installed, per distribution + # metadata (which does not import it)? If so, this is a broken environment. + installed <- tryCatch(isTRUE(nat.python::py_module_info(module)$available[1]), + error = function(e) FALSE) + + list(state = if (installed) "broken" else "absent", error = err) } +# Skip a test when a Python module is unavailable, but fail loudly when it is +# installed yet unimportable -- nat.python's job is to provision a working +# Python, so a broken module must never masquerade as a benign skip. skip_if_no_module <- function(module) { - testthat::skip_if_not(py_module_ok(module), - paste0("Python module '", module, "' not available")) + st <- py_module_status(module) + if (identical(st$state, "ok")) + return(invisible(TRUE)) + if (identical(st$state, "broken")) + stop(sprintf("Python module '%s' is installed but failed to import: %s", + module, st$error), call. = FALSE) + testthat::skip(paste0("Python module '", module, "' not available")) }