CI: provision Python via simple_python(); pin pandas<3

.github/workflows/R-CMD-check.yaml

@@ -1,12 +1,12 @@
 # Single-platform (Ubuntu) R CMD check, including the testthat suite, across a
 # small matrix of Python versions.
 #
-# Python is provided the way real users get it -- through reticulate's managed
-# miniconda -- rather than a bare system Python, so CI exercises the same kind
-# of interpreter nat.python is designed to manage. nat.python itself is installed
-# and used to introspect the environment before the check runs. (Phase 2 will
-# provision Python through nat.python's own environment engine; until then we
-# call reticulate directly.)
+# Python is provided the way real users get it -- through nat.python's own
+# environment engine (simple_python()), which installs miniconda, the
+# r-reticulate env and pandas (via pip). This exercises the exact provisioning
+# path nat.python ships, rather than a hand-rolled reticulate::py_install() call
+# (whose conda-forge pandas could be an interpreter nat.python never gives a
+# user).
 #
 # The matrix spans the Python range we care about on one platform:
 #   * reticulate's own default miniconda Python (what most users get);
@@ -53,19 +53,23 @@ jobs:
           extra-packages: any::rcmdcheck
           needs: check
 
-      - name: Provision Python (reticulate miniconda) and install nat.python
+      - name: Install nat.python and provision Python via simple_python()
         run: |
           v <- Sys.getenv("NATPY_PYTHON_VERSION")
           if (nzchar(v)) {
+            # simple_python() -> install_miniconda() honours this, so it pins the
+            # Python version of the managed r-reticulate environment.
             Sys.setenv(RETICULATE_MINICONDA_PYTHON_VERSION = v)
-            message("Pinning reticulate miniconda Python to ", v)
+            message("Pinning miniconda Python to ", v)
           } else {
             message("Using reticulate's default miniconda Python")
           }
-          reticulate::install_miniconda()
-          reticulate::py_install(c("numpy", "pandas"))
           pak::local_install()
           library(nat.python)
+          # The end-user path: provision miniconda + r-reticulate + pandas (via
+          # pip, pulling a mutually-compatible numpy). "minimal" is nat.python's
+          # own baseline, all pandas2df() needs.
+          simple_python("minimal")
           print(py_module_info(c("numpy", "pandas")))
           py <- reticulate::py_config()$python
           cat(sprintf("RETICULATE_PYTHON=%s\n", py),

.github/workflows/test-coverage.yaml

@@ -1,7 +1,7 @@
 # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
 # Test coverage via covr, uploaded to Codecov. Python is provided the same way
-# R-CMD-check.yaml does -- through reticulate's managed miniconda -- so coverage
-# exercises the same kind of interpreter nat.python is designed to manage.
+# R-CMD-check.yaml does -- through nat.python's own simple_python() engine -- so
+# coverage exercises the exact provisioning path nat.python ships.
 on:
   push:
     branches: [main, master]
@@ -30,16 +30,18 @@ jobs:
           extra-packages: any::covr, any::xml2
           needs: coverage
 
-      - name: Provision Python (reticulate miniconda) and install nat.python
-        # Mirrors R-CMD-check.yaml's default leg: reticulate's own miniconda,
-        # numpy + pandas installed, then pin RETICULATE_PYTHON to that exact
-        # interpreter for the coverage run.
+      - name: Install nat.python and provision Python via simple_python()
+        # Mirrors R-CMD-check.yaml's default leg, but provisions through
+        # nat.python's own simple_python(): miniconda + r-reticulate + pandas
+        # (via pip). pyarrow is added so the use_arrow = TRUE conversion path is
+        # exercised. RETICULATE_PYTHON is then pinned to that interpreter for the
+        # coverage run.
         shell: Rscript {0}
         run: |
-          reticulate::install_miniconda()
-          # pyarrow lets the use_arrow = TRUE conversion path be exercised
-          reticulate::py_install(c("numpy", "pandas", "pyarrow"))
           pak::local_install()
+          library(nat.python)
+          simple_python("minimal", pkgs = "pyarrow")
+          print(py_module_info(c("numpy", "pandas", "pyarrow")))
           cat(sprintf("RETICULATE_PYTHON=%s\n", reticulate::py_config()$python),
               file = Sys.getenv("GITHUB_ENV"), append = TRUE)
 

NEWS.md

@@ -1,3 +1,12 @@
+# nat.python 0.2.0.9000 (development version)
+
+* `simple_python()` now pins the baseline install to `pandas < 3`. pandas 3.0
+  makes Arrow-backed strings the default dtype, which `pandas2df()` does not yet
+  convert back to R; the pin keeps the provisioned environment functional until
+  that support lands (#6).
+* CI now provisions Python through `simple_python()` itself (the end-user path),
+  rather than a bespoke `reticulate::py_install()` call.
+
 # nat.python 0.2.0
 
 First tagged release. A small shared layer of Python interoperability and

R/env.R

@@ -76,8 +76,13 @@ simple_python <- function(pyinstall = c("basic", "full", "extra", "minimal",
   if (pyinstall %in% c("minimal", "basic", "full", "extra")) {
     # nat.python's own baseline: pandas2df() needs pandas, and numpy rides in
     # with it. Every richer bundle builds on top of this.
-    cli::cli_inform("Installing pandas (brings numpy)")
-    ourpip("pandas")
+    #
+    # Pinned to pandas < 3 for now: pandas 3.0 makes Arrow-backed strings the
+    # default dtype (PDEP-14), which pandas2df() does not yet convert back to R
+    # (string columns return as raw ArrowStringArray objects). Lift this once
+    # pandas2df() handles the Arrow string dtype.
+    cli::cli_inform("Installing pandas (<3 for now; brings numpy)")
+    ourpip("pandas<3")
   }
   if (pyinstall %in% c("basic", "full", "extra")) {
     cli::cli_inform("Installing cloudvolume")

tests/testthat/helper-nat.python.R

@@ -1,11 +1,43 @@
-# Can we import a given Python module? Tests that need Python are skipped when
-# it (or the module) is unavailable, so the suite still runs with no Python.
-py_module_ok <- function(module) {
-  requireNamespace("reticulate", quietly = TRUE) &&
-    !inherits(try(reticulate::import(module), silent = TRUE), "try-error")
+# Classify a Python module's availability in the active environment. We keep the
+# three cases apart on purpose (see skip_if_no_module):
+#   * "ok"      -- importable, run the test;
+#   * "absent"  -- no Python, or the module simply isn't installed -> skip;
+#   * "broken"  -- the module *is* installed (per distribution metadata) but
+#                  importing it fails -> a broken environment, not absence.
+# The old helper collapsed "broken" into "absent", so an installed-but-
+# unimportable module (e.g. a conda pandas built against an incompatible numpy)
+# silently skipped and quietly dropped coverage instead of being noticed.
+py_module_status <- function(module) {
+  if (!requireNamespace("reticulate", quietly = TRUE))
+    return(list(state = "absent", error = NA_character_))
+  if (!isTRUE(try(reticulate::py_available(initialize = TRUE), silent = TRUE)))
+    return(list(state = "absent", error = NA_character_))
+
+  # Try importing first: a clean import is the fast path, and it also covers
+  # stdlib/namespace modules that carry no distribution metadata (e.g. datetime).
+  imported <- try(reticulate::import(module), silent = TRUE)
+  if (!inherits(imported, "try-error"))
+    return(list(state = "ok", error = NA_character_))
+
+  err <- conditionMessage(attr(imported, "condition"))
+
+  # Import failed. Is the module nonetheless installed, per distribution
+  # metadata (which does not import it)? If so, this is a broken environment.
+  installed <- tryCatch(isTRUE(nat.python::py_module_info(module)$available[1]),
+                        error = function(e) FALSE)
+
+  list(state = if (installed) "broken" else "absent", error = err)
 }
 
+# Skip a test when a Python module is unavailable, but fail loudly when it is
+# installed yet unimportable -- nat.python's job is to provision a working
+# Python, so a broken module must never masquerade as a benign skip.
 skip_if_no_module <- function(module) {
-  testthat::skip_if_not(py_module_ok(module),
-                        paste0("Python module '", module, "' not available"))
+  st <- py_module_status(module)
+  if (identical(st$state, "ok"))
+    return(invisible(TRUE))
+  if (identical(st$state, "broken"))
+    stop(sprintf("Python module '%s' is installed but failed to import: %s",
+                 module, st$error), call. = FALSE)
+  testthat::skip(paste0("Python module '", module, "' not available"))
 }