From 8e546246753aeb8671646fb750954a5efe2d5db3 Mon Sep 17 00:00:00 2001 From: oskaresparza Date: Fri, 11 Sep 2026 08:41:27 +0200 Subject: [PATCH 1/2] Add %catalog help / %ingest help Lists every public CatalogSession/IngestSession method with its live signature (introspected, so it can't drift from source) and a short description. Works without DREMIO credentials since it's handled before session setup. Co-Authored-By: Claude Sonnet 5 --- src/eea_datalakehouse/notebook/magics.py | 77 ++++++++++++++++++++++++ tests/notebook/test_magics.py | 28 +++++++++ 2 files changed, 105 insertions(+) diff --git a/src/eea_datalakehouse/notebook/magics.py b/src/eea_datalakehouse/notebook/magics.py index f139418..1598d02 100644 --- a/src/eea_datalakehouse/notebook/magics.py +++ b/src/eea_datalakehouse/notebook/magics.py @@ -56,6 +56,7 @@ from __future__ import annotations +import inspect import os from typing import Any @@ -75,6 +76,31 @@ ), } +# One short line per public method — shown by `%catalog help`/`%ingest help`. +# Kept separate from each method's own (much longer) docstring on purpose: +# this is a quick-reference table, not a replacement for reading the real +# docstring in `catalog/session.py`/`dds_ingestion/session.py`. +_CATALOG_HELP = [ + ( + "set_context", + "Set the current path; a later relative path (leading '.') resolves against it.", + ), + ("copy", "Queue a copy. Reversible unless overwrite=True."), + ("move", "Queue a move. Reversible unless overwrite=True."), + ("tag", "Queue replacing a path's tag set."), + ("untag", "Queue removing tags from a path's tag set."), + ("set_wiki", "Queue setting a path's wiki text."), + ("delete_wiki", "Queue deleting a path's wiki text."), + ("set_meta", "Queue setting a folder's Meta Data wiki section."), + ("create_folder", "Queue creating a folder."), + ("delete_folder", "Queue deleting a folder. Never reversible."), + ("commit", "Run every queued step as one all-or-nothing batch."), +] +_INGEST_HELP = [ + ("ingest", "Queue one folder ingest (see FolderIngest for what each argument means)."), + ("commit", "Run every queued ingest in order; stops at the first failure."), +] + def _build_catalog_session() -> CatalogSession: base_url = os.environ.get("DREMIO_BASE_URL") @@ -87,6 +113,51 @@ def _build_catalog_session() -> CatalogSession: return CatalogSession(Catalog(base_url, token, username=username)) +def _is_help(line: str) -> bool: + return line.strip() in ("help", "help()") + + +def _format_signature(func: Any) -> str: + """Render `func`'s signature (minus `self`) the way it reads in source — + `inspect.Signature`'s own `str()` wraps string annotations in quotes + (they're plain `str`s at runtime because of this module's, and the + session modules', `from __future__ import annotations`), which is + accurate but noisy for a notebook help message.""" + sig = inspect.signature(func) + parts = [] + seen_star = False + for name, param in sig.parameters.items(): + if name == "self": + continue + if param.kind is inspect.Parameter.KEYWORD_ONLY and not seen_star: + parts.append("*") + seen_star = True + piece = name + if param.annotation is not inspect.Parameter.empty: + piece += f": {param.annotation}" + if param.default is not inspect.Parameter.empty: + piece += f" = {param.default!r}" + parts.append(piece) + rendered = f"({', '.join(parts)})" + if sig.return_annotation is not inspect.Signature.empty: + rendered += f" -> {sig.return_annotation}" + return rendered + + +def _print_help(cls: type, methods: list[tuple[str, str]], label: str) -> None: + """Print every method in `methods` with its real signature (introspected + from `cls`, so it can't drift from the source) and a one-line + description. Signatures drop `self`; everything else — parameter names, + defaults, `*`-only markers, return types — comes straight from `cls`.""" + print(f"%{label} methods — usage: {_USAGE[label]}") + print() + for name, description in methods: + print(f" {name}{_format_signature(getattr(cls, name))}") + print(f" {description}") + print() + print(f"%{label} help — show this message") + + def _dispatch( session: Any, line: str, user_ns: dict[str, Any], label: str, error_type: type[Exception] ) -> Any: @@ -164,6 +235,9 @@ def _apply_context(self, path: str) -> None: @line_magic def catalog(self, line: str) -> Any: + if _is_help(line): + _print_help(CatalogSession, _CATALOG_HELP, "catalog") + return None if self._catalog_session is None: try: self._catalog_session = _build_catalog_session() @@ -179,6 +253,9 @@ def catalog(self, line: str) -> Any: @line_magic def ingest(self, line: str) -> Any: + if _is_help(line): + _print_help(IngestSession, _INGEST_HELP, "ingest") + return None if self._ingest_session is None: self._ingest_session = IngestSession() return _dispatch( diff --git a/tests/notebook/test_magics.py b/tests/notebook/test_magics.py index 8ec94c4..c8577b0 100644 --- a/tests/notebook/test_magics.py +++ b/tests/notebook/test_magics.py @@ -134,3 +134,31 @@ def test_empty_line_prints_usage(ip: Any, capsys: pytest.CaptureFixture[str]) -> out = capsys.readouterr().out assert "usage:" in out + + +@pytest.mark.parametrize("line", ["help", "help()"]) +def test_catalog_help_lists_methods_without_needing_credentials( + ip: Any, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], line: str +) -> None: + monkeypatch.delenv("DREMIO_BASE_URL", raising=False) + monkeypatch.delenv("DREMIO_TOKEN", raising=False) + + ip.run_line_magic("catalog", line) + + out = capsys.readouterr().out + assert "%catalog methods" in out + assert "copy(source_path: str, target_path: str" in out + assert "commit(*, retry: bool = False" in out + assert "DREMIO_BASE_URL" not in out # never tried to build a session + assert _magics_instance(ip)._catalog_session is None + + +@pytest.mark.parametrize("line", ["help", "help()"]) +def test_ingest_help_lists_methods(ip: Any, capsys: pytest.CaptureFixture[str], line: str) -> None: + ip.run_line_magic("ingest", line) + + out = capsys.readouterr().out + assert "%ingest methods" in out + assert "ingest(folder: str | Path, target_catalog_path: str" in out + assert "commit(*, retry: bool = False, max_retries: int = 3)" in out + assert _magics_instance(ip)._ingest_session is None From 2c99d9c940f1fac518b2c262991130d1dd8ba4ac Mon Sep 17 00:00:00 2001 From: oskaresparza Date: Fri, 11 Sep 2026 08:43:33 +0200 Subject: [PATCH 2/2] Document %catalog/%ingest help() in README and example notebooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README had no section on the notebook facade at all — add one covering %catalog/%ingest usage plus the new help() methods. Add a matching quick-reference cell to each example notebook. Co-Authored-By: Claude Sonnet 5 --- README.md | 25 ++++++++++++++++++++ docs/notebooks/catalog_session_example.ipynb | 14 +++++++++++ docs/notebooks/ingest_session_example.ipynb | 14 +++++++++++ 3 files changed, 53 insertions(+) diff --git a/README.md b/README.md index 91df0f4..e7d178e 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,31 @@ exact same arguments. - `catalog.close()` (or `with Catalog(...) as catalog:`) — disposes the REST/Flight session(s). Idempotent, and covered by a process-exit/SIGTERM fallback if you forget. +## Notebook facade (`%catalog` / `%ingest`) + +For interactive use in JupyterLab, `eea_datalakehouse.notebook` registers two line magics — +thin, queue-then-commit wrappers over `CatalogSession`/`IngestSession` aimed at data +custodians rather than application developers. See +`docs/notebook-facade-for-data-scientists.md` for the full design, and +`docs/notebooks/catalog_session_example.ipynb` / `ingest_session_example.ipynb` for worked +examples. Install the extra this needs once: `pip install "EEADataLakehouse[notebook]"`. + +```python +import eea_datalakehouse.notebook # registers %catalog/%ingest — no %load_ext needed + +%catalog copy("draft.raw_2026", "bwd.reference.water_temperature") +%catalog tag(".water_temperature", ["reviewed"]) +%catalog commit(retry=True) + +%ingest ingest(folder="./bw_2026", target_catalog_path="bwd.reference", + data_format="parquet", table_name="water_temperature") +%ingest commit(retry=True) +``` + +`%catalog help` (or `%catalog help()`) lists every `CatalogSession` method with its +signature and a short description; `%ingest help` does the same for `IngestSession` — handy +when you don't remember an exact parameter name mid-notebook. + ## Layout | Path | Purpose | diff --git a/docs/notebooks/catalog_session_example.ipynb b/docs/notebooks/catalog_session_example.ipynb index b9f067e..8073ef8 100644 --- a/docs/notebooks/catalog_session_example.ipynb +++ b/docs/notebooks/catalog_session_example.ipynb @@ -31,6 +31,20 @@ "source": "`%load_ext eea_datalakehouse.notebook.magics` still works too, and is safe to run either\nbefore or after the import above — whichever runs first registers the magics, the other is\na no-op (see `magics.load_ipython_extension`'s docstring).", "metadata": {} }, + { + "cell_type": "markdown", + "id": "0f5e1ae8", + "source": "## Quick reference\n\n`%catalog help` (or `%catalog help()`) lists every `CatalogSession` method with its\nsignature and a short description — handy when you don't remember an exact parameter\nname mid-notebook. It works even before `DREMIO_BASE_URL`/`DREMIO_TOKEN` are set, since\nit's answered before a session is built.", + "metadata": {} + }, + { + "cell_type": "code", + "id": "0c8132e6", + "source": "%catalog help()", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/docs/notebooks/ingest_session_example.ipynb b/docs/notebooks/ingest_session_example.ipynb index 4a4e74e..1a98729 100644 --- a/docs/notebooks/ingest_session_example.ipynb +++ b/docs/notebooks/ingest_session_example.ipynb @@ -34,6 +34,20 @@ "source": "`%load_ext eea_datalakehouse.notebook.magics` still works too, and is safe to run either\nbefore or after the import above — whichever runs first registers the magics, the other is\na no-op (see `magics.load_ipython_extension`'s docstring).", "metadata": {} }, + { + "cell_type": "markdown", + "id": "53bc1b4a", + "source": "## Quick reference\n\n`%ingest help` (or `%ingest help()`) lists every `IngestSession` method with its signature\nand a short description — handy when you don't remember an exact parameter name\nmid-notebook.", + "metadata": {} + }, + { + "cell_type": "code", + "id": "4ea00f57", + "source": "%ingest help()", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, { "cell_type": "markdown", "metadata": {},