diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..35ba9c2 --- /dev/null +++ b/.flake8 @@ -0,0 +1,14 @@ +[flake8] +extend-ignore = E500, E501 +exclude = + # No need to traverse our git directory + .git, + # There's no value in checking cache directories + __pycache__, + # The conf file is mostly autogenerated, ignore it + docs/source/conf.py, + # This contains our built documentation + build, + # This contains builds of flake8 that we don't want to check + dist +max-complexity = 10 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..f9cf727 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,34 @@ +name: Tests + +on: + push: + branches: [ main, master, cc-development ] + pull_request: + branches: [ main, master, cc-development ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Install package and test dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Run tests with pytest + run: | + pytest -v --tb=short diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e2d1806 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +_* +.* +_*/ +.*/ +*.pyc +__pycache__/ +build/ +dist/ +*.egg-info/ +.pytest_cache/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..42684db --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog + +v0.0.2 +------ +- major refactor +- added tests +- package scaffolding + +v0.0.1 +------ +- initial version diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ad50d3c --- /dev/null +++ b/Makefile @@ -0,0 +1,30 @@ +# Makefile for building and publishing the package +# +# Usage: +# make build - Build source and wheel distributions +# make upload - Upload distributions to PyPI (requires twine credentials) +# make publish - Build and upload in one step +# make clean - Remove build artifacts + +.PHONY: build upload publish clean help + +# Default target +help: + @echo "Available targets:" + @echo " make build - Build sdist and wheel into dist/" + @echo " make upload - Upload dist/* to PyPI via twine" + @echo " make publish - Build then upload" + @echo " make clean - Remove dist/ and build artifacts" + +build: + python -m build + +upload: + python -m twine upload dist/* --verbose + +publish: build upload + +clean: + rm -rf dist/ build/ *.egg-info src/*.egg-info + find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true + find . -type f -name '*.pyc' -delete 2>/dev/null || true diff --git a/README.md b/README.md index 476c72d..a973ebe 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,65 @@ # IPython plugin for DataLinter -> Warning: This work is still very experimental. +[![License](https://img.shields.io/pypi/l/datalintermagic)](https://pypi.org/project/datalintermagic/) +[![Tests](https://github.com/zgornel/Ipython-datalinter/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/zgornel/Ipython-datalinter/actions/workflows/tests.yml?query=branch%3Amain) +[![PyPI version](https://img.shields.io/pypi/v/datalintermagic)](https://pypi.org/project/datalintermagic/) +[![Python versions](https://img.shields.io/pypi/pyversions/datalintermagic)](https://pypi.org/project/datalintermagic/) +![til](./gifs/jupyter.gif) This is a Ipython magic that allows one to run [DataLinter](https://github.com/zgornel/DataLinter) in Jupyter notebooks. -[![License](http://img.shields.io/badge/license-GPL-brightgreen.svg?style=flat)](LICENSE) +## Installation -![til](./gifs/jupyter.gif) +Make sure you have up-to-date stable versions of [IPython](https://ipython.org/) and [Jupyter](https://jupyter.org/). In order for the linter to work, both the Jupyter plugin and DataLinter (server in a Docker image) need to be installed in the system. -## Installation +### Jupyter plugin package -Make sure you have up-to-date stable versions of [IPython](https://ipython.org/) and [Jupyter](https://jupyter.org/). +The Jupyter plugin can be installed with `pip`: +``` +pip install datalintermagic +``` ### DataLinter -Install [DataLinter](https://github.com/zgornel/DataLinter) by pulling the Docker image: -``` +[DataLinter](https://github.com/zgornel/DataLinter) installation is done with: +```bash docker pull ghcr.io/zgornel/datalinter-compiled:latest ``` -and start it with: -``` -docker run -it --rm -p10000:10000 \ +The command to start the linting server is: +```bash +docker run -d --rm -p10000:10000 \ ghcr.io/zgornel/datalinter-compiled:latest \ /datalinterserver/bin/datalinterserver \ -i 0.0.0.0 \ - --config-path /datalinter/config/default.toml \ - --log-level debug + --config-path /datalinter/config/python_modelling_config.toml \ + --log-level info ``` If the server starts correctly, it should display something like: ``` -Warning: KB file not correctly specified, defaults will be used. -└ @ datalinterserver /DataLinter/apps/datalinterserver/src/datalinterserver.jl:83 [ Info: • Data linting server online @0.0.0.0:10000... -[ Info: Listening on: 0.0.0.0:10000, thread id: 1 ``` -### Ipython-datalinter magic -Clone the repository: -``` -git clone https://github.com/zgornel/Ipython-datalinter -``` +### Quick start +```python +%reload_ext datalintermagic +import numpy as np +X = np.random.random((10,100)) -To run the demo, enter the `Ipython-datalinter` folder, run -``` -jupyter notebook +%add --tracked-variable X --data-header --data-delim ',' +%lintline X --show-stats ``` -and open `ipython/demo.ipynb`. -> Note: In order for the `datalintermagics` magic to be available, the `datalintermagics` folder/module needs to be available to the jupyter notebook. The easy way to do this is to have the `datalintermagics` directory and notebooks in the same location. +Further examples of linter usage in Jupyter are found in this [notebook](https://github.com/zgornel/Ipython-datalinter/blob/main/notebooks/workflow_example.ipynb). +## License + +This code has an GNU GPLv3 license. + +## Contributing + +To report a bug or request a feature, please [file an issue](https://github.com/zgornel/Ipython-datalinter/issues/new). + +Recent changes can be found in [CHANGELOG.md](CHANGELOG.md). + +## References + +[1] https://en.wikipedia.org/wiki/Lint_(software) diff --git a/gifs/jupyter.gif b/gifs/jupyter.gif index 0dc62e7..e32cbba 100644 Binary files a/gifs/jupyter.gif and b/gifs/jupyter.gif differ diff --git a/ipython/datalintermagics/lintermagic.py b/ipython/datalintermagics/lintermagic.py deleted file mode 100644 index a2d03eb..0000000 --- a/ipython/datalintermagics/lintermagic.py +++ /dev/null @@ -1,190 +0,0 @@ -import argparse -import shlex -import IPython -import pandas as pd -import json -import http.client -import numpy as np - -from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic) - -TRACKED_VARIABLES = {} - - -@magics_class -class DataLinterMagic(Magics): - tracked_variable = None - data_header = None - data_delim = None - ip = None - port = None - show_stats = None - show_na = None - show_passing = None - - def __parse_add_linter_data_magic(self, line): - try: - parser = argparse.ArgumentParser() - parser.add_argument("--tracked-variable") - parser.add_argument("--data-header") - parser.add_argument("--data-delim") - args = parser.parse_args(shlex.split(line)) - # print(f">>DEBUG: Parsed %%add_linter_data line OK. args={args}") - # tracked variable checks - if args.tracked_variable is None: - print("Warning (Linter): Use '--tracked-variable' to specify a data variable!") # noqa: E501 - return None - tracked_variable = args.tracked_variable - # data header checks - data_header = True if args.data_header == 'True' else False - if data_header is False and args.data_header != 'False' \ - and args.data_header is not None: - print("Warning (Linter): Wrong value for '--data-header', specify True / False. Assuming False...") # noqa: E501 - return None - # data delimiter checks - data_delim = args.data_delim - if data_delim is None: - print("Warning (Linter): Provide a data delimiter for '--data-delim'") # noqa: E501 - return None - return tracked_variable, data_header, data_delim - except Exception as ex: - print(f"Warning (Linter): Could not parse line for %%add_linter_data line magic:\n\t{ex}") # noqa: E501 - return None - - def __parse_lint_magic(self, line): - try: - parser = argparse.ArgumentParser() - parser.add_argument("--ip") - parser.add_argument("--port") - parser.add_argument("--show-stats") - parser.add_argument("--show-na") - parser.add_argument("--show-passing") - args = parser.parse_args(shlex.split(line)) - # print(f">>DEBUG: Parsed %%lint line OK. args={args}") - # IP - if args.ip is None: - print("Warning (Linter): Use '--ip' to specify an ip") - return None - ip = args.ip - # TODO: Implement IP correctness checks - # port - if args.port is None: - print("Warning (Linter): Use '--port' to specify a port") - return None - port = int(args.port) - # TODO: Implement port conversion checks, perhaps default port - # show_stats - show_stats = True if args.show_stats == 'True' else False - if show_stats is False and args.show_stats != 'False' \ - and args.show_stats is not None: - print("Warning (Linter): Wrong value for '--show-stats', specify True / False. Assuming False...") # noqa: E501 - return None - # show_na - show_na = True if args.show_na == 'True' else False - if show_na is False and args.show_na != 'False' \ - and args.show_na is not None: - print("Warning (Linter): Wrong value for '--show-na', specify True / False. Assuming False...") # noqa: E501 - return None - # show_passing - show_passing = True if args.show_passing == 'True' else False - if show_passing is False and args.show_passing != 'False' \ - and args.show_passing is not None: - print("Warning (Linter): Wrong value for '--show-passing', specify True / False. Assuming False...") # noqa: E501 - return None - return ip, port, show_stats, show_na, show_passing - except Exception as ex: - print(f"Warning (Linter): Could not parse line for %%lint cell magic:\n\t{ex}") # noqa: E501 - return None - - @line_magic - def add_linter_data(self, line): - parsed_args = self.__parse_add_linter_data_magic(line) - ipy = IPython.get_ipython() - if parsed_args is not None: - self.tracked_variable, self.data_header, self.data_delim = parsed_args # noqa: E501 - try: - # Support only for: numpy.array and pandas.dataframe - _v = ipy.ev(self.tracked_variable) - if type(_v) is pd.DataFrame or type(_v) is np.ndarray: - TRACKED_VARIABLES[self.tracked_variable] = _v - print(f">>DEBUG: Added '{self.tracked_variable}' as linter data.") # noqa: E501 - else: - print(f"Warning (Linter): Unsupported type {type(_v)} for '{self.tracked_variable}'. Supported: 'numpy.ndarray', 'pandas.DataFrame'.") # noqa: E501 - return None - except Exception as ex: - print(f"Warning (Linter): Variable '{self.tracked_variable}' not found in the current cell:\n\t{ex}") # noqa: E501 - else: - print(f">>DEBUG: '%add_linter_data' line magic FAILED (parsed_args={parsed_args})") # noqa: E501 - return None - - # TODO: Investigate when to execute or provide two different magic commands i.e. @lint_before, @lint_after - @cell_magic - def lint(self, line, cell): - # TODO: Support a '--with-data' option that allows to specify - # which tracked variable to use as data; - # As of now, the behavior is to use the last variable name, - # specified by the `%add_linter_data line magic` - parsed_args = self.__parse_lint_magic(line) - ipy = IPython.get_ipython() - if parsed_args is not None: - self.ip, self.port, self.show_stats, self.show_na, \ - self.show_passing = self.__parse_lint_magic(line) - _v = TRACKED_VARIABLES[self.tracked_variable] - if type(_v) is pd.DataFrame: - _csv_v = self.dataframe_to_csv_string(_v) - elif type(_v) is np.ndarray: - _csv_v = self.ndarray_to_csv_string(_v) - else: - print(f"Warning (Linter): Unsupported type {type(_v)} for '{self.tracked_variable}'\nSupported: 'numpy.ndarray', 'pandas.DataFrame'.") # noqa: E501 - return None - varbody = { - 'linter_input': { - 'context': { - 'data': _csv_v, - 'data_delim': self.data_delim, - 'data_header': self.data_header, - 'code': cell}, - 'options': { - 'show_stats': self.show_stats, - 'show_passing': self.show_passing, - 'show_na': self.show_na} - } - } - jsonbody = json.dumps(varbody) - try: - linter_response_json = self.http_lint_request(jsonbody) - linter_response = json.loads(linter_response_json) - print(f"Linter output\n-------------\n{linter_response['linting_output']}") # noqa: E501 - except Exception as ex: - print(f"Warning (Linter): Failed to read linter output (perhaps linting failed):\n\t{ex}") # noqa: E501 - else: - print(f">>DEBUG: '%%lint' cell magic FAILED (parsed_args={parsed_args})") # noqa: E501 - exec(cell, globals()) - return None - - def ndarray_to_csv_string(self, arr): - df = pd.DataFrame(arr) - return self.dataframe_to_csv_string(df) - - def dataframe_to_csv_string(self, df): - _header = False - _colnames = list(df.columns) - if self.data_header is True and _colnames: - _header = _colnames - elif self.data_header is True and not _colnames: - _header = ['x'+str(i) for i in range(len(df.columns))] - else: - pass - return df.to_csv(None, - sep=self.data_delim, - header=_header, - index=False) - - def http_lint_request(self, body): - host = self.ip - conn = http.client.HTTPConnection(host, self.port) - conn.request("POST", "/api/lint", body=body, ) - response = conn.getresponse() - print(f"Response: {response.status}, {response.reason}") - resp_decoded = response.read().decode() - return resp_decoded diff --git a/ipython/demo.ipynb b/ipython/demo.ipynb deleted file mode 100644 index 3f5df85..0000000 --- a/ipython/demo.ipynb +++ /dev/null @@ -1,136 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 12, - "id": "996c2b3d-bfe9-41e4-89a7-756a427030e0", - "metadata": {}, - "outputs": [], - "source": [ - "# Load the magic\n", - "%reload_ext datalintermagics" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "id": "37998251-5f1b-4b96-afa9-0f4c60994ed1", - "metadata": {}, - "outputs": [], - "source": [ - "from sklearn import datasets\n", - "import pandas as pd\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "iris = datasets.load_iris()\n", - "X = np.array(iris.data)\n", - "#X[:,1]=np.nan\n", - "#print(X)" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "id": "f086c1be-8766-4bfa-803a-fcd16ae44fcc", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - ">>DEBUG: Added 'X' as linter data.\n" - ] - } - ], - "source": [ - "# Add data variable to linter with line magic\n", - "%add_linter_data --tracked-variable X --data-header 'False' --data-delim ','" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "id": "e8a20dc0-c00e-4760-9664-e26989cf8271", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Response: 200, OK\n", - "Linter output\n", - "-------------\n", - "• info \t(enum_detector) \tcolumn: x2 just a few distinct values in 'column: x2', it could be an enum\n", - "• experimental \t(many_missing_values)\tcolumn: x2 found many missing values in 'column: x2'\n", - "2 issues found from 15 linters applied (6 OK, 9 N/A) .\n", - "\n" - ] - } - ], - "source": [ - "%%lint --ip 0.0.0.0 --port 10000 --show-stats True --show-na False --show-passing False\n", - "# This simply lints the data ;)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "fea864ac-a755-4307-b45c-643ada6d323a", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh0AAAGdCAYAAAC2OMGiAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQAALQ1JREFUeJzt3XtwVdXZ+PEnQAiiJAI/EdAMAl4wovws0kIraItg1b7KTNs/HItoHcaxeE2dKnTesfRi/I3t9DJ2qFLHehmlM7VYnYIj7SsXO0SpioChahUFEaReSJBIgHB+82w8vElM9tknZ+111lr7+5k5gzl75WTvdbY5T/Z+nmdV5HK5nAAAAKSsT9o/AAAAgKADAABYw5UOAABgBUEHAACwgqADAABYQdABAACsIOgAAABWEHQAAAAr+ollhw4dkvfee08GDRokFRUVtn88AADoBe0lumfPHhk5cqT06dPHj6BDA47a2lrbPxYAABiwbds2OfHEE/0IOvQKR36nq6urbf94AADQCy0tLdFFg/znuBdBR/6WigYcBB0AAPillNQIEkkBAIAVBB0AAMAKgg4AAGAFQQcAAHAv6DjppJOiBJKuj3nz5qW3hwAAIAhFVa+sW7dO2tvbj3y9adMmmTFjhnz7299OY98AAEBWg47jjjuu09d33XWXjB07Vs477zzT+wUAAALT6z4d+/fvl0ceeUTq6+tja3bb2tqiR8fmIgAAIHt6nUj6xBNPyO7du+Wqq66KHdfQ0CA1NTVHHrRAR3faD+Vk7Zsfyl/Wb4/+1a8BAGGpyOkKLr1w4YUXSv/+/eWpp56KHdfdlQ4NPJqbm+lIisjTm3bIwqeaZEfzviMzMqJmgNzxX3Xy9fEjmCUAcIB+fuvFg1I+v3t1e+Wdd96Rv/3tb/LnP/+54NiqqqroAfQUcFz3yEvSNfLd2bwven7Rd75A4AEAWb698sADD8iwYcPkkksuMb9HyAy9haJXOLq71JZ/TrdzqwUAMhp0HDp0KAo65syZI/36WV8vDgF5YctHnW6pdBd46HYdBwDIYNCht1W2bt0q3/3ud9PZI2TGrj37jI4DALit6EsVM2fOlF7mngKdDBs0wOg4AIDbWHsFZfPF0UOiKpWeurzo87pdxwEA/EfQgbLp26ciKotVXQOP/Ne6XccBAPxH0IGy0j4cWhY7vKbzLRT9mnJZAAgL5SdwIvCYUTc8qlLRpFHN4dBbKlzhAICwEHTACRpgTBk7tNy7AQBIEUEH0ANtSsbVF2QF5ztsIOgAusF6MMgSznfYQiIp0MN6MF27pebXg9HtQCg432ETQQfQAevBIEs432EbQQfQAevBIEs432EbQQfQAevBIEs432EbiaSAZ+vBUGWALJ3vCAtBB9DNejCaNNrdsoYVn3VLLdd6MFQZIEvnO8LD7RXAk/VgqDJAls53hImgA/BgPRiqDJCl8x3h4vYK4MF6MMVUGdBOHr6f7wgXQQfgwXowVBkgS+c7wsXtFcADVBkACAFBB+BRlUFPF7v1ed1OlQEAlxF0AB6gygBACAg6AE9QZQDAdySSAh6hygCAzwg6AM9QZYA00F4fNhB0AEDG0V4ftpDTAQAZRnt92ETQAQAZRXt92EbQAQAZVUx7fcAEgg4AyCja68M2EkkBC6gMgItorw/bCDqAlFEZANfb6+9s3hfdSumuvb4ucU97fZjC7RUgRVQGwGW014dtBB1ASqgMgA9orw+buL0COFAZMGXsUN4HlA3t9WELQQeQEioD4BPa68MGbq8AKaEyAAA6I+gAUq4M0AqA7ujzup3KAABZQdABpITKAADojKADSBGVAQDwv0gkBVJGZQAAHEbQAVhoXW6yMoCW6v7jPURWEXQAHrUud3W/kBzvIbKMnA7Ak9blru4XkuM9RNYRdAAetC53db+QHO8hQNAB9Lp1uU2u7heS4z0ECDoAL1qXu7pfSI73ECCRFPCidbmr+5VGRUaolR0+vIeAc9Ur27dvl9tuu02WL18un376qZx66qly//33y8SJE9PZQ6AMrcs1ObO77Aj96Btehtblru6X6YqMkCs7XH8PAecSST/++GP5yle+IpWVlVHQ0dTUJL/4xS/k2GOPTW8PAYtcbV3u6n6ZrMgIvbLD5fcQsKUil8slTne//fbb5R//+IesWbOm1z+wpaVFampqpLm5Waqrq3v9OkCaXP2L27X90lsh5/6//+kxyTX/1/tzt30t9sPU1Ov4wLX3ELD5+V1U0FFXVycXXnihvPvuu7Jq1So54YQT5Hvf+57MnTu3x+9pa2uLHh13ura2lqADznM1t8Cl/Vr75ody+eLGguMemzs5tiOrqdfxhUvvIWAz6Cgqp+Ott96SRYsWSX19vSxYsEBeeOEFufHGG6WqqkquvPLKbr+noaFBFi5c2KudA8rJZOvyUJmqyMhaZQfnFrKqqKDj0KFDcs4558idd94ZfX322WfLq6++GgUiPQUd8+fPj4KUrlc6APh/ad5URQaVHUA2FJVIOmLEiOgWS0enn366bN26tcfv0asgehmm4wNA8VxMtNTbAscOrIwdo9sLVWTkKzt6usGgz+t2KjuADAUdWrny2muvdXru9ddfl1GjRpneLwCBtNBOkqlAZQeQDUUFHbfccos0NjZGt1f+/e9/y6OPPir33XefzJs3L709BOBsC239ebtbD8SO+bj1QKL90ttDi77zhahKpSP9Wp+nsgPIWE7HpEmTZOnSpVGexo9//GMZPXq0/OpXv5IrrrgivT0E4Gyipen90sBiRt1wKjuAQBXdkfQb3/hG9AAo+7PH1UTLNPaLyg4gXEUHHYCLVRShyydaxt1iKUeiJa29AaSW0wG4WkUROv3rf/wJ8ZVfup327ABcRtCBzFRR+Gz/wUPy9827Ysfodh1nGwmgAJLi9gpSq6Kgm6c5D699WwrFcbpdx10zdYzYRgIogCQIOhBEFUXo3vmo1ei4NJAACqAQgg4EUUURulFDBhod53I1k94i0is2GkDp8cyecpL078edYCAEBB0oCtUK5aEfvD9btjn2Fot+vus4n6uZGpY1yeI1Wzodpx733KmjZf7FnZdgAOAf/nxAUWhXXR76l75+8MbR7eW4ImCqmkkDjntXdw44lH6tz+t2AH4j6EDRqFYoD/1L/9ppo6MrGh3p1/p8Oa4EmKpm0lsqeoUjjm4vR3UOAHO4vYJeoVqhPDSw+P7Mcc7kPJiqZnK9OgeAGQQd6DWqFcpDAwxXPnhNVTP5UJ0DoHQEHXCCi+u4mKyicPH4TOyXqWqmtKpzXJ13IKsIOlB2Lq7jYrKKwsXjM7VfpqqZ0qjOcXXegSwjkRRl5eI6LiarKFw8PpP7ZaqayXR1jqvzDmQdQQfKxsV1XExWUbh4fGnsl6lqJlPVOa7OOwBur6CMXFzHxWQVhYvHl9Z+mapmMlGd4+q8AyDoQBm5uI6LySqKtI6v1ATXtPbLVDVTqdU5Lp5XAA4jkRRl4+I6LiarKNI4PhMJri7Ou0mhHx/gM3I6UDb5yoeeLsDr8yMSVD6YpFcNCt0RSFpFYfr4TCW46s87dmBl7JjBAyutznvo5xWAwwg6UDYuruNisorC5PHZbhPuc4qli+cVgMMIOlBWLq7jYnKNE1PHV0yCayGaQLm79UDsGN2u43zl4nkFgJwOOMDFdVxMrnFi4vh8SHB1jYvnFZB1JJLCCS6u42JyjZNSj8/1BFcASIKgA/CAyTbh+UTLuF4WISRa0gYdcA85HYAHTCe4XjohPqdBt/t8G4I26ICbCDoAT5hsE/7kK/Frj+h2X9uE0wYdcBe3VwCP2GgTLp63CacNOuAugg7AM7QJj5eV6hzARwQdCI5eXqdM0n71iivzTnUO4C6CDgSFigVJXL2ys3lft51HKz5rolVM9YpL857G8QEwg0RSBIOKhfK0CXdt3mmDDriLoANBoGKhPG3CXZ132qADbuL2CoJAxUJ52oS7PO+0QQfcQ9CBIGSpYkFXkjWxJoyJ9uyuz7uL7fWBLCPoQBCyUrHQsKwpWsK+490KbY+u3UiLWf3WlKzMOwAzyOlAEPIVCz3dGKgIYD0RDTjuXd054FD6tT6v223LwrwDMIegA0EIvWJBb6noFY44ul3H2RT6vAMwi6ADwQi5YkFzOAoVgOh2HWdbyPMOwCxyOhCUUCsWNGnU5DjTQp13AGYRdCA4eoth2cb35O0PW+WkoQPl/9YeK0f171vWKpFS6c83Oc611uUAsqEil8tZ7drT0tIiNTU10tzcLNXV1TZ/NDJg7kPrZEXTrs89P6NumCy+clJJVSL6WVyuKhENgMb99/LYWyy6f//6yUWJAyOTrctdaoMOwN3Pb3I6EHzAofR53e5rlYgGEtNPHxY7RrcXE3CYal3uWht0AO4i6EAQPt3f3mPAkafbdZyPVSJ6G2TT9pbYMbo9Sbtxk63LXW2DDsBNBB0Iwp0Jrz4UGudqlUihduPSod24ydblNl8LQPgIOhAETRo1Mc7VKhGT7cZdfS0A4aN6JUWhVwa4VN2hVSpr3kg2rhxVIi61G3f1tQCEr6ig40c/+pEsXLiw03PHH3+87Ny50/R+eS/0bH7X1gBZcHGdPNy4NdG4OBo46XEUqhLRceVoNx53KyNpu/H8a2miZ3eHWfFZYy/brwUgfEX/WXrGGWfIjh07jjw2btyYzp55LPRsfherO7QPh5bFxtHthfp16JUaDZzi6HbbV3T0CtmlE+KDVd2e5EqaydbltEEHUIyif3P269dPhg8ffuRx3HHHFfsSQQs9m9/V6o7o5145qcfAo5g+HXql5tppo6MrGh3p1/p8Oa7k6Pny5CvxwapuT3pemWxdTht0AKnldLzxxhsycuRIqaqqki996Uty5513ypgxY3oc39bWFj06NhcJWTHZ/FPGDhXfFFPdcc3Uns+LtGhgoWWxWqWS70iqt1SK7UiqgcX3Z45zJmelmOqVpOeVydbltEEHYDzo0CDjoYceklNPPVXef/99+elPfypf/vKX5dVXX5WhQ7v/RdfQ0PC5PJCQhZ7N72p1R0caGFx85sgjH6S9DRT0w7duZI38n0FV0euUMwnYh/NK58fHQBqAo0HHRRdddOS/zzzzTJkyZYqMHTtWHnzwQamvr+/2e+bPn99pm17pqK2tlVCFns3vanWH6QRe1xKB0zivXDtGAOEr6Vrx0UcfHQUfesulJ3obRnu0d3yELJ/N39PfxBVFVBm4SG8xVBT4g7+iDNUdJhN4XUwE1vPl2IGVsWN0e9LzysVjBBC+koIOzdXYvHmzjBjBX0VZyebX/T6qMj4/QrfbPj5TCbw+JwInnXGfjxFAhoKOW2+9VVatWiVbtmyR559/Xr71rW9Ft0vmzJmT3h56KORsfk06bC2wfolut9322lQ7blfbeuvP2916IHbMx60HaF0OIJycjnfffVcuv/xy+eCDD6JS2cmTJ0tjY6OMGjUqvT30VKjZ/K4mNJrar9CPz/RrAUBqQceSJUuKevGsCzGb39VEWVP7FfrxmX6tLLX9B1A61l5BUVxte21qv0y2G89CG/Q8KmEAJMEqswgiUdbUfun28SfEV1jp9nIcn4tt0BWVMACSIuhAMImyJvZL27f/ffOu2DG63Xabd1fboFMJA6AY3F5BUImype6Xq23eXW2DHnrbfwBmEXQguETZUvbL1TbvaVWclPoeUgkDoBgEHYCFNu+lVna4WnGSxn5RBQOEi6AD6EDbt/9s2ebYWyx9imzzbqKyw9WKE9PVPlTBAGEjkRToQFeknX76sNg50e1JV641VdnhasWJyaoaqmCA8BF0AF0u7W/a3hI7J7o9SZWI6coOFytOTFXVUAUDZAO3V4CUqkTSqOxwreLE1HxRBQNkA1c6gIytceLiMbo6VwDM4koH4NEaJyYSLV08RlfXvAFgFlc6gA70VsWxAytj52TwwMrEa5yYei2TiZb5ipOebshU9GIdl1Jfy+Q+AXAXQQdQpFwZXstkoqXJShiTa964uKYPALMIOoAuCY27Ww/Ezolu13G2Xytp8qft9XNMvZara/oAMIecDsDxJEvTr5XG+jmmXsvVNX0AmEHQkaLQ2zmHeHwuJlmafi3X188xuU8hnqOAzwg6UhJ6O+dQj89kW29XXysr54OL+wRkHTkdKQi9nXPIx2eyrbfp1xp/QnXsGN1ejr/iXTwfXNwnAAQdxoXezjkLx2eirbfp19p/8JD8ffOu2DG6Xcdl/XxwcZ8AHMaVDsNMVxm4JuvHJ0Ucn8nXenjt27Er3yrdruOyfj64uE8ADiPoMCz0ds4cX/J5MDlX73zUmui1ko4L+XxwcZ8AHEYiqWFpVRm4koXP8SWfB5NzNWrIwESvlXScqfPKxfblLu4TgMMIOgzLVxlowlp3V8MrPmt2VEyVgUtZ+Bxf8vfP5FzNnnKS/GzZ5thbLBor6Dib51Ua50OpXNwnAIdxe8Uw0+2cXcvC5/jstwhX/fv1kemnD4sdo9t1nM3zysX25S7uE4DDCDpSYKqds6tZ+Byf/Rbh+h5v2t4SO0a3J62qMXleudi+3MV9AiBSkcvlrH5itbS0SE1NjTQ3N0t1dXzfAd+Ver987ZsfyuWLGwuOe2zu5LJ0leT4/JyrtM4rV/KOXN8nwFcmPr/J6XC4nbPrWfgcn59zldZ5FXpLdQClI+hwWOhZ+KEfX0fatEt7aGhJq1aYaMJn0vwLX9ZxAYBCCDocFnoWfujHl9ewrEkWr9nSqfJEK1HmTh0t8y8+nPBoc66yMu8A3EMiqcNCz8IP/fjyAce9qzsHHEq/1ud1u+25ysK8A3ATQYfjQs/CD/n49JaKXuGIo9uTrpdicq5CnncA7qJ6xROhZ+GHeHz3r3lLfvLXzQXH/fclp8s1U8eUZa5CnHcA6aB6JUNCz8I3eXyufJCmtV6KHl/Te81HklInjhrc6+ML/bwC4BYSSREUl1rGp7FeiomkVAAoF3I6EAzXWsZrWWxFgQsQFUWsl2IqKRUAyoWgA0FwsWW83ro4qrJv7JiBlX0T3RoxnZQKAOVA0IEgaA5H1yscHWmoodt1nM19at3fHjtm7/72RPukjcUKxUu6XccBgKsIOhAEF1vGm9yntJJSAcAmEkkRhDRae5daBWNyn9JISgUA2wg6EIR8a++4WywjimjtbaIKxmS7cU021SqVuFssfYpISgWAcuD2CoKgVyAunRAfDOj2JFcqTFXBmGw3rovDTT99WOwY3V7MInIAYBu/oRAEvRXy5CvxwYBuL1S9YroKxlS7cf15m7a3xI7R7TarcwCgWNxeQSaqV6RD9UpcB85iqmCSdvLUwGJG3fCS8kNMHR8AlBNBB4JgqlIkrSqYUtuNu1idAwBWb680NDRIRUWF3HzzzaW8TLD0UvfaNz+Uv6zfHv1byqVvk68V4vGZqhRJowrGBFf3CwCsXOlYt26d3HfffXLWWWf19iWCZnINEJfWE0ljn1yqFDFdBWOKyUoYAPDqSscnn3wiV1xxhSxevFgGDx5sfq88Z3INENfWEzG9T65VipisgjHJZCUMAHgVdMybN08uueQSueCCC8zvkedMVj+4uJ6Iy8dnolLEVBVMGkxVwgCAN7dXlixZIi+99FJ0eyWJtra26JHX0hJf9uc7k9UPaVRShH58pVaKuF4lYqISBgC8CDq2bdsmN910kzzzzDMyYMCAxMmmCxculKwwWWXgYsWCD8enVyGa3muO1iHRtuATRw1O/KHs4pybroQBAC+CjhdffFF27dolEydOPPJce3u7rF69Wu65557oikbfvp2X8p4/f77U19d3utJRW1sroTJZZeBixYLrx9ewrCla4r3j3Q9tHz536miZf3FdWfYJANCLoGP69OmycePGTs9dffXVMm7cOLnttts+F3Coqqqq6JEVJqsMXKxY0J917MBK2d16oMcxgwdWluX4NOC4d/WWzz2vAUj++UKBh4tzDgCZTCQdNGiQjB8/vtPj6KOPlqFDh0b/DbNVBr5WLCRNsTR5fPsPHoqucMTR7TrO1j4BADpj7RXHqwxcq1jQBMa4qxxKt+s4m8f38Nq3Y1dgVbpdx9naJwCA4TboK1euLPUlgmSyysClioU0Ei1NHJ8mjZoc59KcA0AoWHvFkyoDU6+llR2lfJCmlWhZ6vFplYrJcSb2CQDQGUFHhrjUbty02VNOiqpU4m6xaGyl4wAA5UFOR0a41m7ctP79+kRlsXF0u44DAJQHv4EzwMV242nQcthrp42Ormh0pF/r80n6dAAA0lORy+WsLiKhzcFqamqkublZqqurbf7ozNKl4i9f3Fhw3GNzJxeVw1BqfkhatCxWq1TyHUn1lgpXOACg/J/f5HRkQFqtvV1NtNQA45qpY8q9GwCALri9kgG09gYAuICgIwPyFSdxdDutvQEAaSLoyAC9DXLphPjkTt3uQj4GACBcBB0ZoAmfT74SXxKr25NWrwAA0BsEHRmgFSZd+3N0pduTrpcCAEBvUL2SAWlVr0C8Ly8GAJsIOjKA6hX/288DQAi4vZKh6pWe/q7W56lecbv9PACEgKAjA1xdLyV0ptvPA4DvCDoywtX1UrKcwKuhBgm8ALKEnI4M0cBiRt1wEhotIYEXAAIMOqgM8H+9lBDfQxJ4ASCwoIPKAP+F+h7mE3g1abS7rI2Kz25v0X4eQFZ4ndNBZYD/Qn4PSeAFgECCDioD/JeF95AEXgAI4PZKMZUBLuYwIDvvIQm8AOB50EFlgP+y9B66msALADZ5e3uFygD/8R4CQLZ4G3TQ2tt/vIcAkC3eBh1UBviP9xAAssXboENRGeA/3kMAyI6KXC5ntR6xpaVFampqpLm5Waqrq428ZojdLLOG9xAA3Gbi89vb6pWsVQaY+lB29cM9C+8hAGRdEEFH6Ey1CQ+13TgAwA9e53Rkgak24SG3GwcA+IGgIwNtwrPQbhwA4D6CjkDahNt4HQAASkHQkYE24VlqNw4AcBdBRwbahNNuHADgAoKODLQJp904AMAFBB0ZaBNOu3EAgAsIOjLSJpx24wCAcguiDXoWhN6RFADgNtqgZ4ipNuG0GwcAlAu3VwAAgBUEHQAAwAqCDgAAYAVBBwAAsIKgAwAAWEHQAQAA3As6Fi1aJGeddVbUX0MfU6ZMkeXLl6e3d3Ca9vxY++aH8pf126N/9WsAAHrST4pw4oknyl133SUnn3xy9PWDDz4ol112mbz88styxhlnFPNS8NzTm3bIwqeaZEfz/65Mq+vAaFv2pF1SAQDZUnJH0iFDhsjdd98t11xzTaLxdCQNI+C47pGXpOuJk+9rWkx7dgCAH0x8fvc6p6O9vV2WLFkie/fujW6zIBv0Fope4eguUs0/p9u51QIAKOn2itq4cWMUZOzbt0+OOeYYWbp0qdTVHV4JtTttbW3Ro2OkBH/pui0db6l0F3jodh1nom07ACAcRV/pOO2002T9+vXS2Ngo1113ncyZM0eampp6HN/Q0BBdjsk/amtrS91nlJEuFGdyHAAgO0rO6bjgggtk7Nixcu+99ya+0qGBB6vM+kmrVC5f3Fhw3GNzJ3OlAwAC0mIgp6Po2ytdaczSMajoqqqqKnogDF8cPSSqUtnZvK/bvA5NJh1eMyAaBwBAr2+vLFiwQNasWSNvv/12lNvxwx/+UFauXClXXHFFMS8Dj/XtUxGVxXasVsnLf63bdRwAAL0OOt5//32ZPXt2lNcxffp0ef755+Xpp5+WGTNmFPMy8JyWw2pZrF7R6Ei/plwWAJBaTkex6NMRDi2L1SoVTRodNujwLRWucABAmFpcyOlAdmmAQVksACApFnwDAABWEHQAAAArCDoAAIAVBB0AAMAKgg4AAGAFQQcAALCCoAMAAFhB0AEAAKwg6AAAAFYQdAAAACsIOgAAgBUEHQAAwAqCDgAAYAVBBwAAsIKgAwAAWEHQAQAArCDoAAAAVhB0AAAAKwg6AACAFQQdAADACoIOAABgBUEHAACwgqADAABYQdABAACsIOgAAABWEHQAAAArCDoAAIAVBB0AAMAKgg4AAGAFQQcAALCCoAMAAFhB0AEAAKwg6AAAAFYQdAAAACsIOgAAgBUEHQAAwAqCDgAAYAVBBwAAsIKgAwAAWEHQAQAArCDoAAAAVhB0AAAAKwg6AACAFQQdAADACoIOAADgXtDR0NAgkyZNkkGDBsmwYcNk1qxZ8tprr6W3dwAAIJtBx6pVq2TevHnS2NgoK1askIMHD8rMmTNl79696e0hAAAIQkUul8v19pv/85//RFc8NBiZNm1aou9paWmRmpoaaW5ulurq6t7+aAAAYJGJz++Scjr0B6shQ4aU8jIAACAD+vX2G/UCSX19vZx77rkyfvz4Hse1tbVFj46REgAAyJ5eBx3XX3+9bNiwQZ577rmCyacLFy7s7Y8BgF5rP5STF7Z8JLv27JNhgwbIF0cPkb59KphR5go+5XTccMMN8sQTT8jq1atl9OjRsWO7u9JRW1tLTgeAVD29aYcsfKpJdjTvO/LciJoBcsd/1cnXx49g9pkrlCGno6igQ4dqwLF06VJZuXKlnHLKKUX/QBJJAdgIOK575CXp+sstf41j0Xe+QODBXMH1RFItl33kkUfk0UcfjXp17Ny5M3p8+umnvfrhAJDGLRW9wtHdX1P553S7jss65gq2FRV0LFq0KIpwzj//fBkxYsSRxx//+Mf09hAAiqA5HB1vqXSloYZu13FZx1zB6UTSElp6AIAVmjRqclzImCvYxtorAIKiVSomx4WMuYJtBB0AgqJlsVql0lNhrD6v23Vc1jFXsI2gA0BQtA+HlsWqroFH/mvdTr8O5gr2EXQACI724dCy2OE1nW+h6NeUyzJX8HTBt96gTwcAW+hIylzBrc/vXrdBBwDX6S2UKWOHlns3vMBcwQaCDgDBXp3gSgfzDrcQdAAIcr0U1l5h3uEecjoABLdeCmuvMO8IYO0VAHB9DRDWE2He4S6CDgBBrQHCeiLMO9xFTgfgGVcTLUt5LZNrgKS1nkiISak+zDvCQtABeMTVRMtSX8vkGiBprCcSalKq6/OO8HB7BfBEPjmy622Inc37oud1u6+vZXINEB0zsH/f2DFH9++beO0Vk3PlGp2DYwdWxo7R7UnnnTVvUAhBB+ABVxMtTb2WyfVS9Gd9eqA9dkzrgXaSUhNKegOJNW+QBEEH4AFXEy1Nvpap9VIeXvu2FFrcQbfruKwnpep+7249EDvm49YDiY+PNW9QCDkdgAdcTfgznTyoH1oz6oaXlLD5zketxsaFnhyZxvGZeA8RLoIOwAOuJvylkTxY6hogo4YMNDbOh+TIT/e3y53LmuTtD1vlpKEDZcHFdXJUgZyWtI/P1XVcQqxA8u34CDoAD+ST9DR5sbs7BxWf3YYoJuEv7rZBMUmbpvbLlNlTTpKfLdsscWkk+ntYx9mcqzTMfWidrGjadeTrNW+IPNy4VWbUDZPFV07y/vhMCrUCybfjI6cD8IDJJD0dc9yg/rFjdHvS1zK1X6b079dHaoccFTtGt+u4QnS/L50Q/wtbt5fjr8muAUdH+rxu9/n4TAq5Asm34yPoADxhKklPL8dveLcldoxu13E298sU3e93Pvw0doxuT3J8ern6yVfif2Hr9iSVPibpvvcUcOTp9kLH6OrxmRR6W3zfjo/bK4BHTCTp6f3/pON+MutMa/tlisnjK1S9Ih2qV2zmMJg6RlePz6RiKpB8PMYXPDs+gg7AM6Um6WnCoclxriUPmjw+V6tXTB2jq8dnUujHuMuz4yPoADzLKi+lWkHp92jCYZJx5TjGUl/H5PGlVd2x/+ChqE+Ilu1qFY0mtSbJMTF9jD5U55QqjWN0qUpkmGfvIUEH4FFWeanVCkqDFP2eJONsH6OJ1zF5fGlUdzQsa5LFa7Z0qq7Rapu5U0fL/IRzbuoYXaw+Ms30MbpWJfJFz95DEkkBT7LKTVQrKL0qokFKHN2e9OqJqWM09Tq636OGxlev6PYkx6d/vY4/oTp2jG5P+leuBhz3ru4ccCj9Wp/X7TbfQxerj0wzeYwuVon09ew9JOgAPMgqN1WtkKdXRXr60CrmqompYzQ5V3rrYttH8dUrul3HJXmtv2+On3fdnvS19ApHHN2e5LVMvoeuVR+lwcQxulwl8nWP3kNurwAeZJWnUXGiH0ql5oeYOkaTc6W5EoV+7+t2HXfN1DFevpbJ99C16qO0lHqMrleJfN2T95CgwxMuJS7BflZ5WhUn+uGUNEhJ8xhNzpXJtVdcfS2T72Ea1Ueu/r4q5Rh9qBLp60gFWRyCDg+4lrgE+1nlaVWcuHKMJufK5Norrr6Wy0L9feVblYiryOlwnIuJSyguq7ynv+8qiqh8SFpJUkzFiQm670cXuJR/dFXfgsdocq60/LTQH9VJ115JMibpOJP75aqQf1+ZPEezjKDDYS4nLsFuVrn2cKgq0MdBtxfT68EEPfdaCySvtra1FzxHTc+Vlp/G0e1J56rQT0x608D0frkm9N9XvlWJuMrPszsjiklcgptMZZXre9xWoKpBt9s+FzTpsdBHSO6zcTYz8LXfxbXTRn/uyoJ+rc8n7Ydh8vhM7peLsvD7yqcqEVeR0+EwHxKXYCer3NVzwXRypMkMfP0A//7McSV1/kwj+dPEfrnI1XM0q1UiriLocBiJS+EoNavc1XMhjeRIkxn4+kGetPzUZvJnqfvlorTOURcrYXyoEnEVQYfDfGtvi+ydC/oXurbwjrtN73NyZOjH5/o5GmolTJb5fT0vcCQuwfVzQf9in356fDtu3e7rrYPQkz9dPkdDroTJMv5PcRyJS3D5XNBL35u2t8SO0e2+ViyEnvzp6jkaeiVMllXkcjmr71pLS4vU1NRIc3OzVFfHL6QEt+9rojxcOhfWvvmhXL64seC4x+ZO9v4eeKnL0WdJqedols4rn5j4/CanwxMkLsHFcyErFQuhJn+6eo5m6bzKGoIOwDMuXelwtaoGfuO8ChdBB+AR17L5Xa2qgd84r8LFDUnAEy5m87taVQO/cV6Fi6AD8IDL2fwuVtXAf5xXYeL2ChDYuhblSDKlNTQ4r5AEQQfgAR+y+V2qqnGdS8nAruO8ynjQsXr1arn77rvlxRdflB07dsjSpUtl1qxZ6ewdgAjZ/OFwLRkYcDqnY+/evTJhwgS555570tkjAD1m8/f0t7A+r9upEnGbi8nAgNNXOi666KLoAcB+Nr9+MGmA0TFdlCqRMJKB9X3U7bpsOrdaEKrUq1fa2tqi1qkdHwCKRzZ/dpKBgVClnkja0NAgCxcuTPvHAJlAlYi/fEgGBrwPOubPny/19fVHvtYrHbW1tWn/WCBYZPP7iWRgwELQUVVVFT0AIMto7Q3QkRQArKC1N9CLoOOTTz6R9evXRw+1ZcuW6L+3bt3KfAJADJKBkXUVuVyuqMUaVq5cKV/96lc/9/ycOXPkD3/4Q8Hv15yOmpoaaW5ulurq6uL2FgACQEdS+MjE53fROR3nn3++FBmnAAA6IBkYWcUqswAAwAqCDgAAYAVBBwAAsIKgAwAAWEHQAQAArCDoAAAAVhB0AAAAKwg6AACAFQQdAAAgjFVmu8p3M9V2qgAAwA/5z+1SupJbDzr27NkT/VtbW2v7RwMAAAOf47oGi5UF30p16NAhee+992TQoEFSUVFhNALTQGbbtm0sJGcR814ezDvzniWc727Mu4YLGnCMHDlS+vTp48eVDt3RE088MbXX14lh9Vr7mPfyYN6Z9yzhfC//vPf2CkceiaQAAMAKgg4AAGBFMEFHVVWV3HHHHdG/YN5Dx/nOvGcJ53s48249kRQAAGRTMFc6AACA2wg6AACAFQQdAADACoIOAABghRdBR0NDg0yaNCnqYjps2DCZNWuWvPbaawW/b9WqVTJx4kQZMGCAjBkzRn73u99Z2d9Q9GbeV65cGXWa7fr417/+ZW2/fbdo0SI566yzjjTkmTJliixfvjz2ezjX7c8753p6v3f0d8bNN98cO45z3u6cmzrfvQg69OSaN2+eNDY2yooVK+TgwYMyc+ZM2bt3b4/fs2XLFrn44otl6tSp8vLLL8uCBQvkxhtvlMcff9zqvvusN/Oep8HJjh07jjxOOeUUK/scAu3Ye9ddd8k///nP6PG1r31NLrvsMnn11Ve7Hc+5Xp55z+NcN2fdunVy3333RcFfHM55+3Nu7HzPeWjXrl1a5ptbtWpVj2N+8IMf5MaNG9fpuWuvvTY3efJkC3sYpiTz/uyzz0ZjPv74Y6v7FrrBgwfnfv/733e7jXO9PPPOuW7Wnj17cqecckpuxYoVufPOOy9300039TiWc97+nJs637240tFVc3Nz9O+QIUN6HLN27dror/KOLrzwwugvmAMHDqS+jyFKMu95Z599towYMUKmT58uzz77rIW9C1N7e7ssWbIkurqkl/u7w7lennnP41w3Q6+qXnLJJXLBBRcUHMs5b3/OTZ3v1hd8K5X2Mquvr5dzzz1Xxo8f3+O4nTt3yvHHH9/pOf1abxF88MEH0aTB/LzrvOqlOs2laWtrk4cffjg6OfV+4LRp05jyhDZu3Bh92O3bt0+OOeYYWbp0qdTV1XU7lnO9PPPOuW6OBngvvfRSdKk/Cc55+3Nu6nz3Lui4/vrrZcOGDfLcc88VHKtJLh3lm692fR7m5v20006LHnn6C1yXRf75z39O0FEEncP169fL7t27ozykOXPmRDk2PX0Acq7bn3fOdTP098NNN90kzzzzTJT0nxTnvN05N3W+e3V75YYbbpAnn3wyuqSjSV9xhg8fHkXDHe3atUv69esnQ4cOTXlPw1LMvHdn8uTJ8sYbb6Syb6Hq37+/nHzyyXLOOedEmeUTJkyQX//6192O5Vwvz7x3h3O9eC+++GL0u1n/gtbfz/rQQO83v/lN9N96q6srznn7c27qfPfiSodeodAPPr3UqZdyRo8eXfB7NAp76qmnOj2nUZ3+MqmsrExxb8PRm3nvjlYPcTur9PdCL2l2h3O9PPPeHc714ukler2t1dHVV18t48aNk9tuu0369u37ue/hnLc/58bO95wHrrvuulxNTU1u5cqVuR07dhx5tLa2Hhlz++2352bPnn3k67feeis3cODA3C233JJramrK3X///bnKysrcn/70pzIdhX96M++//OUvc0uXLs29/vrruU2bNkXb9TR7/PHHy3QU/pk/f35u9erVuS1btuQ2bNiQW7BgQa5Pnz65Z555JtrOue7GvHOup6drJQXnfPnn3NT53s+Xpj3q/PPP7/T8Aw88IFdddVX031ovvHXr1iPb9K/yZcuWyS233CK//e1vZeTIkdGlo29+85uW995fvZn3/fv3y6233irbt2+Xo446Ss444wz561//GvVMQTLvv/++zJ49O5rbmpqaqH7+6aeflhkzZnQ755zr5Zl3znV7OOftS+t8Z2l7AABghVeJpAAAwF8EHQAAwAqCDgAAYAVBBwAAsIKgAwAAWEHQAQAArCDoAAAAVhB0AAAAKwg6AACAFQQdAADACoIOAABgBUEHAAAQG/4/VRrO9BY93ocAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.scatter(X[:,1], X[:,2])\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "de6538bc-0391-43df-8e6b-c0cbecbbb93b", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.7" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/ipython/test_linter_magic.ipynb b/ipython/test_linter_magic.ipynb deleted file mode 100644 index acbe62a..0000000 --- a/ipython/test_linter_magic.ipynb +++ /dev/null @@ -1,281 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7a669577-bb46-47b0-a699-5f4a06817c83", - "metadata": {}, - "source": [ - "# Some context on what's being developed\n", - "### Remarks:\n", - " - A minimal setup should be implemented first\n", - " - Should point to a more extensive mini-language approach (for the future)\n", - "\n", - "### Linting workflow through magics:\n", - "- First, load up the magics:\n", - "```ipython\n", - "%reload_ext datalintermagics \n", - "```\n", - "- **Data setup** (line magic) i.e. , specify dataset plus arguments for passing it to the linter it i.e. `header`, `delimiter`:\n", - "```ipython\n", - "%add_linter_data --tracked-variable my_data_var --data-delim=',' --data-header=True\n", - "```\n", - "- **Lint command** (cell magic) i.e. send code and the tracked variable to the linter, get reply back and show it:\n", - "```ipython\n", - "%%lint --ip 0.0.0.0 --port 10000 --show-stats True --show-na False --show-passing False\n", - "```\n", - " \n", - "### Shortlist of TODOs concerning magics:\n", - " - track more than one variable (defining meaning there)\n", - " - add support for linting lines i.e." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "a66d5b9f-fbd1-4708-96ba-b39a88b83d50", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Warning (Linter): Unsupported type for 'b'. Supported: 'numpy.ndarray', 'pandas.DataFrame'.\n" - ] - } - ], - "source": [ - "%reload_ext datalintermagics\n", - "b = 2\n", - "%add_linter_data --tracked-variable b --data-header True --data-delim ','" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "bbe49aec-65d1-4365-aba1-a78f1f2a05f6", - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "# Though the following import is not directly being used, it is required\n", - "# for 3D projection to work with matplotlib < 3.2\n", - "import mpl_toolkits.mplot3d # noqa: F401\n", - "import numpy as np\n", - "import pandas as pd\n", - "from sklearn import datasets\n", - "from sklearn.cluster import KMeans\n", - "np.random.seed(5)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "b4760889-d106-4a64-9c6b-9d7af210a921", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - ">>DEBUG: Added 'iris_df' as linter data.\n" - ] - } - ], - "source": [ - "from sklearn import datasets\n", - "iris = datasets.load_iris()\n", - "iris_df = pd.DataFrame(\n", - " np.hstack(\n", - " (iris.data, \n", - " np.reshape(iris.target,(len(iris.target),1))\n", - " )\n", - " ))\n", - "iris_df.columns = [*iris.feature_names, 'target']\n", - "\n", - "# Add data variable to linter with line magic\n", - "%add_linter_data --tracked-variable iris_df --data-header 'True' --data-delim ','" - ] - }, - { - "cell_type": "markdown", - "id": "8df2994e-3c0b-41ee-839b-dcdb0a580836", - "metadata": {}, - "source": [ - "### Lint with %%lint cell magic" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "714d0a11-8bfa-425f-9094-e546b2beb517", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Response: 200, OK\n", - "Linter output\n", - "-------------\n", - "• info \t(enum_detector) \tcolumn: target just a few distinct values in 'column: target', it could be an enum\n", - "1 issue found from 18 linters applied (6 OK, 12 N/A) .\n", - "\n" - ] - } - ], - "source": [ - "%%lint --ip 0.0.0.0 --port 10000 --show-stats True --show-na False --show-passing False\n", - "# This simply lints the data, no code is available for the cell" - ] - }, - { - "cell_type": "markdown", - "id": "0f49eaf7-f25d-4b0b-9eaa-89148ec1aaa9", - "metadata": {}, - "source": [ - "### Lint (data and code simultaneously)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "c8a220dc-822f-44b1-b965-9cd725a351d1", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Response: 200, OK\n", - "Linter output\n", - "-------------\n", - "• info \t(enum_detector) \tcolumn: target just a few distinct values in 'column: target', it could be an enum\n", - "1 issue found from 18 linters applied (6 OK, 12 N/A) .\n", - "\n" - ] - } - ], - "source": [ - "%%lint --ip 0.0.0.0 --port 10000 --show-stats True --show-na False --show-passing False\n", - "# This will send code as well to the linter\n", - "X = iris.data\n", - "y = iris.target\n", - "\n", - "%%add_linter_data\n", - "estimators = [\n", - " (\"k_means_iris_8\", KMeans(n_clusters=8)),\n", - " (\"k_means_iris_3\", KMeans(n_clusters=3)),\n", - " (\"k_means_iris_bad_init\", KMeans(n_clusters=3, n_init=1, init=\"random\")),\n", - "]\n", - "\n", - "fig = plt.figure(figsize=(10, 8))\n", - "titles = [\"8 clusters\", \"3 clusters\", \"3 clusters, bad initialization\"]\n", - "for idx, ((name, est), title) in enumerate(zip(estimators, titles)):\n", - " ax = fig.add_subplot(2, 2, idx + 1, projection=\"3d\", elev=48, azim=134)\n", - " est.fit(X)\n", - " labels = est.labels_\n", - "\n", - " ax.scatter(X[:, 3], X[:, 0], X[:, 2], c=labels.astype(float), edgecolor=\"k\")\n", - "\n", - " ax.xaxis.set_ticklabels([])\n", - " ax.yaxis.set_ticklabels([])\n", - " ax.zaxis.set_ticklabels([])00\n", - " ax.set_xlabel(\"Petal width\")\n", - " ax.set_ylabel(\"Sepal length\")\n", - " ax.set_zlabel(\"Petal length\")\n", - " ax.set_title(title)\n", - "\n", - "# Plot the ground truth\n", - "ax = fig.add_subplot(2, 2, 4, projection=\"3d\", elev=48, azim=134)\n", - "\n", - "for name, label in [(\"Setosa\", 0), (\"Versicolour\", 1), (\"Virginica\", 2)]:\n", - " ax.text3D(\n", - " X[y == label, 3].mean(),\n", - " X[y == label, 0].mean(),\n", - " X[y == label, 2].mean() + 2,\n", - " name,\n", - " horizontalalignment=\"center\",\n", - " bbox=dict(alpha=0.2, edgecolor=\"w\", facecolor=\"w\"),\n", - " )\n", - "\n", - "ax.scatter(X[:, 3], X[:, 0], X[:, 2], c=y, edgecolor=\"k\")\n", - "\n", - "ax.xaxis.set_ticklabels([])\n", - "ax.yaxis.set_ticklabels([])\n", - "ax.zaxis.set_ticklabels([])\n", - "ax.set_xlabel(\"Petal width\")\n", - "ax.set_ylabel(\"Sepal length\")\n", - "ax.set_zlabel(\"Petal length\")\n", - "ax.set_title(\"Ground Truth\")\n", - "\n", - "plt.subplots_adjust(wspace=0.25, hspace=0.25)\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "fa634784-60d0-40ab-8717-6e37a7893e98", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - ">>DEBUG: Added 'X' as linter data.\n" - ] - } - ], - "source": [ - "# Add data variable to linter with line magic\n", - "X = iris.data\n", - "%add_linter_data --tracked-variable X --data-header 'False' --data-delim ','" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "1497de30-1a56-4a44-96c9-68c8db309471", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Response: 200, OK\n", - "Linter output\n", - "-------------\n", - "0 issues found from 18 linters applied (6 OK, 12 N/A) .\n", - "\n" - ] - } - ], - "source": [ - "%%lint --ip 0.0.0.0 --port 10000 --show-stats True --show-na False --show-passing False\n", - "##" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.7" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/workflow_example.ipynb b/notebooks/workflow_example.ipynb new file mode 100644 index 0000000..aa4102d --- /dev/null +++ b/notebooks/workflow_example.ipynb @@ -0,0 +1,2824 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7a669577-bb46-47b0-a699-5f4a06817c83", + "metadata": {}, + "source": [ + "# Demo notebook for the DataLinter Jupyter plugin\n", + "\n", + "### Linting workflow:\n", + "1. First, load up the magics:\n", + "```ipython\n", + "%reload_ext datalintermagic \n", + "```\n", + "2. Track specific data variables with `%add`. The magic supports header and delimiter options:\n", + " - header with `--data-header` (if not specified, a default header will be added by the linter)\n", + " - delimiter with `--data-delim: str` (default: `','`)\n", + "```ipython\n", + "%add --tracked-variable my_data_var --data-delim=',' --data-header\n", + "```\n", + "3. Lint specific data variables with `%lintline`.\n", + "```ipython\n", + "%lintline my_data_var\n", + "```\n", + "4. Lint cells with `%%lintcell`. This sends code and all tracked variables to the linter:\n", + "```ipython\n", + "%%lintcell\n", + "# code goes here\n", + "```\n", + "Both `%lintline` and `%%lintcell` support arguments to specify the linter ip and port as well as statistics over the linting and linters that were not available or successful. The additional arguments can be added inline, as below:\n", + "```ipython\n", + "--ip 127.0.0.1 --port 10000 --show-stats --show-na --show-passing\n", + "```\n", + "\n", + "\n", + "> Note: The magic’s display is a side effect, not the return value" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "a66d5b9f-fbd1-4708-96ba-b39a88b83d50", + "metadata": {}, + "outputs": [], + "source": [ + "%reload_ext datalintermagic" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "bbe49aec-65d1-4365-aba1-a78f1f2a05f6", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "# Though the following import is not directly being used, it is required\n", + "# for 3D projection to work with matplotlib < 3.2\n", + "import mpl_toolkits.mplot3d # noqa: F401\n", + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn import datasets\n", + "from sklearn.cluster import KMeans\n", + "from sklearn.linear_model import LogisticRegression\n", + "np.random.seed(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b4760889-d106-4a64-9c6b-9d7af210a921", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/zgornel/projects/Ipython-datalinter/src/datalintermagic/datalintermagic.py:66: DataLinterWarning: Could not add 'f' to linter data: KeyError('f')\n", + " warnings.warn(f\"{msg}\", category=DataLinterWarning)\n" + ] + } + ], + "source": [ + "from sklearn import datasets\n", + "iris = datasets.load_iris()\n", + "iris_df = pd.DataFrame(\n", + " np.hstack(\n", + " (iris.data, \n", + " np.reshape(iris.target,(len(iris.target),1))\n", + " )\n", + " ))\n", + "iris_df.columns = [*iris.feature_names, 'target']\n", + "\n", + "# Add data variable to linter with line magic\n", + "%add --tracked-variable iris_df --data-header --data-delim ','\n", + "%add --tracked-variable f --data-header --data-delim ','" + ] + }, + { + "cell_type": "markdown", + "id": "2fe67aaa-13ab-49b9-8440-511c615e5e18", + "metadata": {}, + "source": [ + "### Lint a specific variable with %%lintline" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1ccdb200-67c8-4ba9-8370-6dedbd87f6f0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\u001b[1m\u001b[30mDataLinter:\n", + "\u001b[0m" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "\u001b[1m\u001b[30m! warning \t(vif_colinearity) \tdataset High multicolinearity detected in dataset using VIF\n", + "! warning \t(cnc_colinearity) \tdataset High multicolinearity detected, condition number=186.86611144578282\n", + "• info \t(int_as_float) \tcolumn='target' the values of 'column='target'' are floating point but can be integers\n", + "Total of 9 linters: 3 Pass, 3 Fail, 3 N/A\n", + "\u001b[0m" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%lintline iris_df --show-stats" + ] + }, + { + "cell_type": "markdown", + "id": "8df2994e-3c0b-41ee-839b-dcdb0a580836", + "metadata": {}, + "source": [ + "### Lint with %%lint cell magic\n", + "The cell has no code and it is impossible to know which data to use. Most linters will be N/A in this case" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "714d0a11-8bfa-425f-9094-e546b2beb517", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\u001b[1m\u001b[30mDataLinter:\n", + "\u001b[0m" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "\u001b[1m\u001b[30mTotal of 5 linters: 0 Pass, 0 Fail, 5 N/A\n", + "\u001b[0m" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%%lintcell --ip 127.0.0.1 --port 10000 --show-stats --show-passing\n", + "# This simply lints the data, no code is available for the cell\n" + ] + }, + { + "cell_type": "markdown", + "id": "40fbad51-20aa-4c2b-8ad5-0202b66f61b3", + "metadata": {}, + "source": [ + "### Prepare to lint a cell with code\n", + "Add additional variables to be tracked" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "c8a220dc-822f-44b1-b965-9cd725a351d1", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/zgornel/projects/Ipython-datalinter/src/datalintermagic/datalintermagic.py:66: DataLinterWarning: Column names of 'X' will be auto-generated as x0,x1...\n", + " warnings.warn(f\"{msg}\", category=DataLinterWarning)\n", + "/home/zgornel/projects/Ipython-datalinter/src/datalintermagic/datalintermagic.py:66: DataLinterWarning: Column names of 'y' will be auto-generated as x0,x1...\n", + " warnings.warn(f\"{msg}\", category=DataLinterWarning)\n" + ] + } + ], + "source": [ + "# This will send code as well to the linter\n", + "X = iris.data\n", + "y = iris.target\n", + "%add --tracked-variable X --data-header --data-delim ','\n", + "%add --tracked-variable y --data-header --data-delim ','\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "1497de30-1a56-4a44-96c9-68c8db309471", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\u001b[1m\u001b[30mDataLinter:\n", + "\u001b[0m" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "\u001b[1m\u001b[30m✓ pass \t(Python_imbalanced_target_variable)\tdataset Target variable values are balanced\n", + "Total of 5 linters: 1 Pass, 0 Fail, 4 N/A\n", + "\u001b[0m" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/lib/python3.14/site-packages/sklearn/linear_model/_logistic.py:599: ConvergenceWarning: lbfgs failed to converge after 100 iteration(s) (status=1):\n", + "STOP: TOTAL NO. OF ITERATIONS REACHED LIMIT\n", + "\n", + "Increase the number of iterations to improve the convergence (max_iter=100).\n", + "You might also want to scale the data as shown in:\n", + " https://scikit-learn.org/stable/modules/preprocessing.html\n", + "Please also refer to the documentation for alternative solver options:\n", + " https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n", + " n_iter_i = _check_optimize_result(\n" + ] + }, + { + "data": { + "text/html": [ + "
LogisticRegression()
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "LogisticRegression()" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
LogisticRegression()
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "LogisticRegression()" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%lintcell --show-passing --show-stats\n", + "model = LogisticRegression();\n", + "model.fit(X, y);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "367d7b03-f2f4-4f43-92d0-e1c5f479f3f3", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c217d02 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = ["setuptools >= 45", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "datalintermagic" +version = "0.0.3" +description = "Jupyter magic for DataLinter" +readme = "README.md" +license = "GPL-3.0-or-later" +license-files = ["LICENSE"] +authors = [{name = "Corneliu Cofaru", email = "cornel@oxoaresearch.com"}] +keywords = ["datalinter", "ipython", "magic", "linting"] +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", +] +requires-python = ">=3.12" +dependencies = ["pandas", "numpy", "IPython", "termcolor"] + +[project.optional-dependencies] +dev = ["pytest>=7.0"] + +[project.urls] +Homepage = "https://github.com/zgornel/Ipython-datalinter" +"Bug Tracker" = "https://github.com/zgornel/Ipython-datalinter/issues" + +[tool.setuptools] +packages = ["datalintermagic"] # Auto-discovery alternative: use find = {where = ["src"]} +package-dir = {"" = "src"} + +[tool.setuptools.package-data] +"datalintermagic" = ["*.txt", "*.json"] # Include non-Python files if needed diff --git a/ipython/datalintermagics/__init__.py b/src/datalintermagic/__init__.py similarity index 64% rename from ipython/datalintermagics/__init__.py rename to src/datalintermagic/__init__.py index 8decda9..b18f404 100644 --- a/ipython/datalintermagics/__init__.py +++ b/src/datalintermagic/__init__.py @@ -1,8 +1,5 @@ """IPython magic for DataLinter""" -__version__ = '0.0.1' - - -from .lintermagic import DataLinterMagic +from .datalintermagic import DataLinterMagic def load_ipython_extension(ipython): diff --git a/src/datalintermagic/datalintermagic.py b/src/datalintermagic/datalintermagic.py new file mode 100644 index 0000000..2508c52 --- /dev/null +++ b/src/datalintermagic/datalintermagic.py @@ -0,0 +1,558 @@ +"""IPython magic commands for interacting with a DataLinter server. + +This module provides line and cell magics that let Jupyter/IPython users +track pandas DataFrames or NumPy arrays and send them (as CSV) to a running +DataLinter HTTP service for linting. + +Typical usage in a notebook:: + + %load_ext datalintermagic + %add --tracked-variable df --data-header + %lintline df --show-stats + %%lintcell --show-na + # cell code that uses df +""" + +import socket +import warnings +import ipaddress +import argparse +import shlex +import pandas as pd +import json +import http.client +import numpy as np + +from IPython.core.magic import Magics, magics_class, line_magic, cell_magic +from IPython.display import display, Pretty +from termcolor import colored + + +DEFAULT_IP = "127.0.0.1" +DEFAULT_PORT = 10000 +DEFAULT_TIMEOUT = 300 +DEFAULT_DELIMITER = ',' + +# New exceptions +class LinterConnectionError(Exception): + """Raised when the client cannot reach the DataLinter server or receives invalid JSON. + + This covers connection failures, timeouts, and responses that are not valid JSON. + """ + + def __init__(self, message): + super().__init__(message) + + +class LinterHTTPError(Exception): + """Raised when the DataLinter server returns a non-200 HTTP status. + + Attributes + ---------- + status : int + HTTP status code. + reason : str + HTTP reason phrase. + body : str + Response body (decoded text). + """ + + def __init__(self, status, reason, body): + super().__init__(body) + self.status = status + self.reason = reason + self.body = body + + +class DataLinterWarning(UserWarning): + """Warning category used for all non-fatal messages emitted by the magics.""" + pass + + +def _parse_delimiter(value, default=','): + """Return a validated delimiter string. + + Parameters + ---------- + value : str or None + User-supplied delimiter, or None to use the default. + default : str, optional + Fallback delimiter (default: ','). + + Returns + ------- + str + The delimiter to use. + + Raises + ------ + ValueError + If *value* is neither None nor a string. + """ + if value is None: + return default + if isinstance(value, str): + return value + raise ValueError(f"expected str, got {value!r}") + + +@magics_class +class DataLinterMagic(Magics): + """IPython magic class that tracks data variables and sends them to DataLinter. + + Line magics + ----------- + %add + Register a DataFrame or ndarray for later linting. + %lintline + Lint a single previously registered variable. + + Cell magic + ---------- + %%lintcell + Lint all currently tracked variables against the cell's source code, + then execute the cell. + """ + + def __init__(self, shell): + """Initialize the magic and its argument parsers. + + Parameters + ---------- + shell : IPython.core.interactiveshell.InteractiveShell + The IPython shell instance. + """ + super().__init__(shell) + self.all_tracked_variables = {} # tracked variables, converted to CSV strings + # Create %add parser + self.parser_add = argparse.ArgumentParser(exit_on_error=False, add_help=False) + self.parser_add.add_argument("--tracked-variable") + self.parser_add.add_argument("--data-header", action='store_true') + self.parser_add.add_argument("--data-delim") + # Create %lintline and %lintcell parser + self.parser_lint = argparse.ArgumentParser(exit_on_error=False, add_help=False) + self.parser_lint.add_argument("tracked_variable", nargs='?', default=None) # used only when linting lines + self.parser_lint.add_argument("--ip") + self.parser_lint.add_argument("--port") + self.parser_lint.add_argument("--show-stats", action='store_true') + self.parser_lint.add_argument("--show-na", action='store_true') + self.parser_lint.add_argument("--show-passing", action='store_true') + + @staticmethod + def display_warning(msg): + """Emit a non-fatal warning via the warnings system. + + Parameters + ---------- + msg : str + Warning message. + + Returns + ------- + None + """ + warnings.warn(f"{msg}", category=DataLinterWarning) + return None + + @staticmethod + def display_output(results): + """Pretty-print linting results in the notebook. + + Parameters + ---------- + results : str + The linting output text returned by the server. + + Returns + ------- + None + """ + header = "DataLinter:\n" + display(Pretty(colored(f"{header}", "black", attrs=["bold"]))) + display(Pretty(colored(f"{results}", "black", attrs=["bold"]))) + return None + + def _parse_add_magic(self, line): + """Parse the arguments of a %add magic line. + + Parameters + ---------- + line : str + The raw line after the magic name. + + Returns + ------- + tuple or None + (tracked_variable, data_header, data_delim) on success, or None + if parsing failed (a warning is emitted). + """ + try: + args = self.parser_add.parse_args(shlex.split(line)) + # tracked variable checks + if args.tracked_variable is None: + DataLinterMagic.display_warning("Use '--tracked-variable' to specify a data variable!") + return None + elif str.isidentifier(args.tracked_variable) is False: + DataLinterMagic.display_warning("Ivalid identifier specified with '--tracked-variable'") + return None + else: + tracked_variable = args.tracked_variable + # data header checks + data_header = args.data_header + # data delimiter checks + data_delim = _parse_delimiter(args.data_delim) + return tracked_variable, data_header, data_delim + except Exception as ex: + DataLinterMagic.display_warning(f"Error in %add magic line parser: {repr(ex)}") + return None + + def _parse_lint_magic(self, line): + """Parse the arguments of a %lintline or %%lintcell magic line. + + Parameters + ---------- + line : str + The raw line after the magic name. + + Returns + ------- + tuple or None + (tracked_variable, ip, port, show_stats, show_na, show_passing) + on success, or None if parsing failed (a warning is emitted). + """ + try: + args = self.parser_lint.parse_args(shlex.split(line)) + tracked_variable = args.tracked_variable + # ip + try: + _ip = ipaddress.ip_address(args.ip) if args.ip is not None else DEFAULT_IP + ip = str(_ip) # return string + except Exception as exc: + # Not legal + raise ValueError(f"The IP '{args.ip!r}' is not valid") from exc + # port + try: + port = int(args.port) if args.port is not None else DEFAULT_PORT + assert 1 <= port <= 65535, "Invalid port value. Needs to be an integer between 1 and 65535" + except Exception as ex: + raise ValueError(f"{ex}") + show_stats = args.show_stats + show_na = args.show_na + show_passing = args.show_passing + return tracked_variable, ip, port, show_stats, show_na, show_passing + except Exception as ex: + DataLinterMagic.display_warning(f"Line parse error: {repr(ex)}") + return None + + @line_magic + def add(self, line): + """Register a variable for linting. + + The variable must be a pandas DataFrame or a NumPy ndarray present in + the user namespace. It is converted to a CSV string and stored together + with its header and delimiter settings. + + Parameters + ---------- + line : str + Magic arguments, e.g. ``--tracked-variable df --data-header --data-delim ;`` + + Returns + ------- + None + """ + parsed_args = self._parse_add_magic(line) + if parsed_args is not None: + tracked_variable, data_header, data_delim = parsed_args + try: + # Support only for: numpy.array and pandas.dataframe + v = self.shell.user_ns[tracked_variable] + v_delimited = self.to_csv_string(v, header=data_header, delimiter=data_delim, symbol=tracked_variable) + self.all_tracked_variables[tracked_variable] = (v_delimited, data_header, data_delim) + except Exception as ex: + DataLinterMagic.display_warning(f"Could not add '{tracked_variable}' to linter data: {repr(ex)}") + else: + DataLinterMagic.display_warning("Could not parse line for '%add' magic") + return None + + @line_magic + def lintline(self, line): + """Lint a single previously registered variable. + + The variable's current value is re-read from the user namespace (if it + still exists) before being sent to the server. + + Parameters + ---------- + line : str + Magic arguments, e.g. ``df --ip 127.0.0.1 --port 10000 --show-stats`` + + Returns + ------- + None + """ + parsed_args = self._parse_lint_magic(line) + if parsed_args is not None: + # Build request body for linter + tracked_variable, ip, port, show_stats, show_na, show_passing = parsed_args + if tracked_variable is not None and tracked_variable in self.all_tracked_variables.keys(): + # Update data variable from local namespace (uses the same delimiter, header) + if tracked_variable in self.shell.user_ns: + _, _header, _delim = self.all_tracked_variables[tracked_variable] + v = self.shell.user_ns[tracked_variable] + self.all_tracked_variables[tracked_variable] = (self.to_csv_string(v, _header, _delim, tracked_variable), _header, _delim) + _data, _header, _delim = self.all_tracked_variables[tracked_variable] + varbody = { + 'linter_input': { + 'context': { + 'data': _data, + 'data_delim': _delim, + 'data_header': _header, + 'code': ''}, + 'options': { + 'show_stats': show_stats, + 'show_passing': show_passing, + 'show_na': show_na} + } + } + jsonbody = json.dumps(varbody).encode("utf-8") + try: + linter_response = self.http_lint_request(ip, port, jsonbody) + DataLinterMagic.display_output(linter_response['linting_output']) + except LinterConnectionError as ex: + DataLinterMagic.display_warning(f"Cannot reach server: {repr(ex)}") + except LinterHTTPError as ex: + DataLinterMagic.display_warning( + f"Server rejected the request " + f"(HTTP {ex.status} {ex.reason}):\n\t{ex.body}" + ) + except Exception as ex: + DataLinterMagic.display_warning(f"Failed to read linter output: {repr(ex)}") + else: + DataLinterMagic.display_warning("'%lintline' magic needs a valid data symbol") + else: + DataLinterMagic.display_warning("Could not parse line for '%lintline' magic") + return None + + # Note: It would be interesting to Investigate when to execute the linter (before of after cell execution) + # One could provide two different magic commands i.e. @lint_before, @lint_after + # Currently, linting is done BEFORE cell execution and is non-blocking + @cell_magic + def lintcell(self, line, cell): + """Lint all tracked variables against the cell source, then execute the cell. + + Linting is non-blocking: the cell always runs, even if parsing or the + HTTP request fails. All tracked variables are refreshed from the user + namespace before the request is sent. + + Parameters + ---------- + line : str + Magic arguments (IP, port, display flags). The optional positional + variable name is ignored. + cell : str + The source code of the cell. + + Returns + ------- + Any + The result of executing the cell (``shell.run_cell(...).result``). + """ + parsed_args = self._parse_lint_magic(line) + # Build request body for linter + if parsed_args is not None: + _, ip, port, show_stats, show_na, show_passing = parsed_args + # Check again for variable changes (re-bindings) and update all variables + for tv in self.all_tracked_variables.keys(): + if tv in self.shell.user_ns: + _, _header, _delim = self.all_tracked_variables[tv] + v = self.shell.user_ns[tv] + self.all_tracked_variables[tv] = (self.to_csv_string(v, _header, _delim, tv), _header, _delim) + varbody = { + 'linter_input': { + 'context': { + 'data': self.all_tracked_variables, + # 'data_delim' and 'data_header' are not passed, each variable has its own + 'code': cell}, + 'options': { + 'show_stats': show_stats, + 'show_passing': show_passing, + 'show_na': show_na} + } + } + jsonbody = json.dumps(varbody).encode("utf-8") + try: + linter_response = self.http_lint_request(ip, port, jsonbody) + DataLinterMagic.display_output(linter_response['linting_output']) + except LinterConnectionError as ex: + DataLinterMagic.display_warning(f"Cannot reach server: {repr(ex)}") + except LinterHTTPError as ex: + DataLinterMagic.display_warning( + f"Server rejected the request " + f"(HTTP {ex.status} {ex.reason}):\n\t{ex.body}" + ) + except Exception as ex: + DataLinterMagic.display_warning(f"Failed to read linter output, perhaps linting failed: {repr(ex)}") + else: + DataLinterMagic.display_warning("Could not parse line for '%lintcell' magic") + # Always run cell, regardless of linter status + # Also, one can run with `self.shell.ex(cell)` if one only wants side effects + result = self.shell.run_cell(cell) + return result.result + + def to_csv_string(self, v, header, delimiter, symbol=None): + """Convert a DataFrame or ndarray to a CSV string. + + Parameters + ---------- + v : pandas.DataFrame or numpy.ndarray + The value to convert. + header : bool + Whether to include a header row. + delimiter : str + Field separator. + symbol : str, optional + Variable name (used in warning messages). + + Returns + ------- + str + CSV representation. + + Raises + ------ + TypeError + If *v* is neither a DataFrame nor an ndarray. + """ + if isinstance(v, pd.DataFrame): + return self.dataframe_to_csv_string(v, header, delimiter, symbol=symbol) + elif isinstance(v, np.ndarray): + return self.ndarray_to_csv_string(v, header, delimiter, symbol=symbol) + else: + raise TypeError(f"Unsupported type '{type(v)}' for delimited-string transformation") + + def ndarray_to_csv_string(self, arr, header, delimiter, symbol=None): + """Convert a NumPy ndarray to a CSV string via a temporary DataFrame. + + Parameters + ---------- + arr : numpy.ndarray + Input array. + header : bool + Whether to include a header row. + delimiter : str + Field separator. + symbol : str, optional + Variable name (used in warning messages). + + Returns + ------- + str + CSV representation. + """ + df = pd.DataFrame(arr) + return self.dataframe_to_csv_string(df, header=header, delimiter=delimiter, symbol=symbol) + + def dataframe_to_csv_string(self, df, header, delimiter, symbol=None): + """Convert a pandas DataFrame to a CSV string suitable for DataLinter. + + Non-string column names cause a warning and are replaced by generated + names (x0, x1, ...). Missing values are written as the string ``NA``. + + Parameters + ---------- + df : pandas.DataFrame + Input frame. + header : bool + Whether to include a header row. + delimiter : str + Field separator. + symbol : str, optional + Variable name (used in warning messages). + + Returns + ------- + str + CSV representation. + """ + _header = None + _colnames = list(df.columns) + _str_colnames = _colnames and all(isinstance(col, str) for col in _colnames) + if header is True and _str_colnames: + _header = _colnames + elif header is True and not _str_colnames: + DataLinterMagic.display_warning(f"Column names of '{symbol}' will be auto-generated as x0,x1...") + _header = ['x'+str(i) for i in range(len(df.columns))] + else: + # header is False, header not exported to csv + pass + return df.to_csv(None, + sep=delimiter, + header=_header, + index=False, + na_rep="NA") # Linter supports ["", "NA", "NaN", "N/A", "NAN"] + + def http_lint_request(self, ip, port, body, timeout=DEFAULT_TIMEOUT, path="/api/lint"): + """Send a JSON POST request to the DataLinter server and return the parsed response. + + Wildcard bind addresses (``0.0.0.0``, ``::``, ``[::]``) are rewritten to + the loopback address for the actual connection. The HTTP connection + timeout is always ``DEFAULT_TIMEOUT``; the *timeout* argument is only + used in the timeout error message. + + Parameters + ---------- + ip : str + Server IP address. + port : int or str + Server port. + body : bytes + JSON request body. + timeout : float, optional + Timeout value used only in the error message (default: DEFAULT_TIMEOUT). + path : str, optional + HTTP path (default: ``/api/lint``). + + Returns + ------- + dict + Parsed JSON response. + + Raises + ------ + LinterConnectionError + On connection failure, timeout, or invalid JSON. + LinterHTTPError + On any non-200 HTTP status. + """ + connect_host = DEFAULT_IP if ip in {"0.0.0.0", "::", "[::]"} else ip + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + "Content-Length": str(len(body)), + } + conn = http.client.HTTPConnection(connect_host, int(port), timeout=DEFAULT_TIMEOUT) + try: + conn.request("POST", path, body=body, headers=headers) + response = conn.getresponse() + raw = response.read() + status, reason = response.status, response.reason + except socket.timeout as ex: + raise LinterConnectionError( + f"Timed out after {timeout}s talking to {connect_host}:{port}" + ) from ex + except (ConnectionRefusedError, socket.gaierror, OSError) as ex: + raise LinterConnectionError( + f"Could not reach DataLinter at {connect_host}:{port} ({ex}). " + "Is the Docker server running?" + ) from ex + finally: + conn.close() + try: + text = raw.decode("utf-8", errors="replace") + if status != 200: + raise LinterHTTPError(status, reason, text) + parsed = json.loads(text) + return parsed + except json.JSONDecodeError as ex: + raise LinterConnectionError(f"Invalid JSON from server: {ex}") from ex diff --git a/tests/test_datalintermagic.py b/tests/test_datalintermagic.py new file mode 100644 index 0000000..436bf0f --- /dev/null +++ b/tests/test_datalintermagic.py @@ -0,0 +1,762 @@ +"""Tests for ``datalintermagic.datalintermagic``. + +Regenerated to match the current source on the cc-development branch. + +Maintainability notes: +- Message fragments are centralized and match the wording in the source exactly + (including the typo "Ivalid"). +- Warnings are emitted via warnings.warn(..., category=DataLinterWarning). +- Display output uses IPython.display.display + Pretty + termcolor.colored. +- Parsing errors return None and emit a warning (they do not raise). +- Only valid Python identifiers are accepted for tracked variables. +- Cell magic always executes the cell, even if parsing or linting fails. +- HTTP connection timeout is always DEFAULT_TIMEOUT (the timeout argument is + only used in the timeout error message). +""" + +import json +import socket +import sys +import warnings +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +# Ensure local src is importable when running from a checkout +_SRC = Path(__file__).resolve().parents[1] / "src" +if _SRC.is_dir() and str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) + +from datalintermagic.datalintermagic import ( # noqa: E402 + DEFAULT_DELIMITER, + DEFAULT_IP, + DEFAULT_PORT, + DEFAULT_TIMEOUT, + DataLinterMagic, + DataLinterWarning, + LinterConnectionError, + LinterHTTPError, + _parse_delimiter, +) + +# --------------------------------------------------------------------------- +# Centralized expected message fragments (update here if source wording changes) +# --------------------------------------------------------------------------- +MSG_MISSING_TRACKED = "Use '--tracked-variable' to specify a data variable!" +MSG_INVALID_IDENTIFIER = "Ivalid identifier specified with '--tracked-variable'" +MSG_ADD_PARSE_ERROR = "Error in %add magic line parser:" +MSG_LINE_PARSE_ERROR = "Line parse error:" +MSG_ADD_FAIL = "Could not add" +MSG_ADD_PARSE_FAIL = "Could not parse line for '%add' magic" +MSG_LINTLINE_NEEDS_SYMBOL = "'%lintline' magic needs a valid data symbol" +MSG_LINTLINE_PARSE_FAIL = "Could not parse line for '%lintline' magic" +MSG_LINTCELL_PARSE_FAIL = "Could not parse line for '%lintcell' magic" +MSG_CANNOT_REACH = "Cannot reach server:" +MSG_HTTP_REJECT = "Server rejected the request" +MSG_FAILED_READ = "Failed to read linter output" +MSG_FAILED_READ_CELL = "Failed to read linter output, perhaps linting failed" +MSG_GENERATED_NAMES = "will be auto-generated as x0,x1..." +MSG_OUTPUT_HEADER = "DataLinter:" + +# Small reusable CSV fixtures +TWO_COL_NO_HEADER = "1,3\n2,4\n" +TWO_COL_HEADER = "a,b\n1,3\n2,4\n" +TWO_COL_PIPE_HEADER = "a|b\n1|3\n2|4\n" + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- +@pytest.fixture(scope="module") +def shell(): + import IPython + from IPython.core.interactiveshell import InteractiveShell + + ipython = InteractiveShell.instance() + assert IPython.get_ipython() is ipython + return ipython + + +@pytest.fixture +def magic(shell): + m = DataLinterMagic(shell) + m.all_tracked_variables.clear() + return m + + +@pytest.fixture +def two_col_frame(): + return pd.DataFrame({"a": [1, 2], "b": [3, 4]}) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +def capture_display(monkeypatch): + """Monkeypatch display() to collect Pretty (or other) objects as strings. + + ANSI colour codes from termcolor are stripped so assertions stay simple. + """ + displayed = [] + + def fake_display(obj, *args, **kwargs): + text = getattr(obj, "data", None) + if text is None: + text = str(obj) + # Strip common ANSI escape sequences + import re + text = re.sub(r"\x1b\[[0-9;]*m", "", text) + displayed.append(text) + + monkeypatch.setattr("datalintermagic.datalintermagic.display", fake_display) + return displayed + + +def capture_warnings(): + """Context manager that records warning messages of category DataLinterWarning.""" + return warnings.catch_warnings(record=True) + + +def assert_any_warning_contains(warning_list, fragment): + messages = [str(w.message) for w in warning_list if issubclass(w.category, DataLinterWarning)] + assert any(fragment in msg for msg in messages), ( + f"No warning containing {fragment!r}. Got: {messages}" + ) + + +def assert_any_display_contains(displayed, fragment): + assert any(fragment in text for text in displayed), ( + f"No display containing {fragment!r}. Got: {displayed}" + ) + + +def _install_http(monkeypatch): + """Replace HTTPConnection with a recording fake. Returns a mutable state dict.""" + state = { + "response": None, + "request_error": None, + "response_error": None, + "conn": None, + } + + class FakeResponse: + def __init__(self, status, reason, body, content_type="application/json"): + self.status = status + self.reason = reason + self._body = body if isinstance(body, bytes) else body.encode("utf-8") + self._content_type = content_type + + def read(self): + return self._body + + def getheader(self, name, default=None): + if str(name).lower() == "content-type": + return self._content_type + return default + + class FakeConn: + def __init__( + self, + host, + port=None, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None, + blocksize=8192, + ): + self.host = host + self.port = port + self.timeout = timeout + self.closed = False + self.requested = None + state["conn"] = self + + def request(self, method, url, body=None, headers=None, *, encode_chunked=False): + self.requested = { + "method": method, + "url": url, + "body": body, + "headers": dict(headers or {}), + } + if state["request_error"] is not None: + raise state["request_error"] + + def getresponse(self): + if state["response_error"] is not None: + raise state["response_error"] + return state["response"] + + def close(self): + self.closed = True + + state["FakeResponse"] = FakeResponse + monkeypatch.setattr( + "datalintermagic.datalintermagic.http.client.HTTPConnection", + FakeConn, + ) + return state + + +def _capture_lint(monkeypatch, magic, response=None): + """Monkeypatch http_lint_request to record the call and return a canned response.""" + if response is None: + response = {"linting_output": "ok"} + captured = {} + + def fake_http(ip, port, body, timeout=DEFAULT_TIMEOUT, path="/api/lint"): + captured["ip"] = ip + captured["port"] = port + captured["body"] = body + captured["timeout"] = timeout + captured["path"] = path + return response + + monkeypatch.setattr(magic, "http_lint_request", fake_http) + return captured + + +def make_line_payload(data, delim=",", header=False, show_stats=False, + show_passing=False, show_na=False): + return { + "linter_input": { + "context": { + "data": data, + "data_delim": delim, + "data_header": header, + "code": "", + }, + "options": { + "show_stats": show_stats, + "show_passing": show_passing, + "show_na": show_na, + }, + } + } + + +def make_cell_payload(tracked, cell, show_stats=False, show_passing=False, show_na=False): + # tracked values are tuples; JSON will turn them into lists + return { + "linter_input": { + "context": { + "data": tracked, + "code": cell, + }, + "options": { + "show_stats": show_stats, + "show_passing": show_passing, + "show_na": show_na, + }, + } + } + + +def assert_payload_equals(body_bytes, expected_dict): + assert json.loads(body_bytes.decode()) == expected_dict + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- +class TestExceptions: + def test_connection_error_message(self): + err = LinterConnectionError("cannot connect") + assert isinstance(err, Exception) + assert str(err) == "cannot connect" + + def test_http_error_keeps_status_reason_and_body(self): + err = LinterHTTPError(503, "Service Unavailable", "overloaded") + assert isinstance(err, Exception) + assert err.status == 503 + assert err.reason == "Service Unavailable" + assert err.body == "overloaded" + assert str(err) == "overloaded" + + def test_datalinter_warning_is_userwarning(self): + assert issubclass(DataLinterWarning, UserWarning) + + +class TestDefaults: + def test_constant_values(self): + assert DEFAULT_IP == "127.0.0.1" + assert DEFAULT_PORT == 10000 + assert DEFAULT_TIMEOUT == 300 + assert DEFAULT_DELIMITER == "," + + def test_magic_names_are_registered(self): + assert DataLinterMagic.magics["line"]["add"] == "add" + assert DataLinterMagic.magics["line"]["lintline"] == "lintline" + assert DataLinterMagic.magics["cell"]["lintcell"] == "lintcell" + + +class TestParseDelimiter: + def test_none_uses_default(self): + assert _parse_delimiter(None) == "," + assert _parse_delimiter(None, default="|") == "|" + + def test_string_is_returned_unchanged(self): + assert _parse_delimiter(";") == ";" + assert _parse_delimiter("") == "" + assert _parse_delimiter(" ; ") == " ; " + + @pytest.mark.parametrize("value", [5, b",", 1.5, ("a",)]) + def test_non_string_is_rejected(self, value): + with pytest.raises(ValueError, match="expected str"): + _parse_delimiter(value) + + +class TestParseAddMagic: + @pytest.mark.parametrize( + "line,expected", + [ + ("--tracked-variable df", ("df", False, ",")), + ("--tracked-variable=df", ("df", False, ",")), + ( + "--data-delim ; --tracked-variable df --data-header", + ("df", True, ";"), + ), + ("--tracked-variable df --data-delim ''", ("df", False, "")), + ], + ) + def test_success_valid_identifier(self, magic, line, expected): + with capture_warnings() as rec: + result = magic._parse_add_magic(line) + assert result == expected + assert not any(issubclass(w.category, DataLinterWarning) for w in rec) + + def test_missing_tracked_variable(self, magic): + with capture_warnings() as rec: + result = magic._parse_add_magic("") + assert result is None + assert_any_warning_contains(rec, MSG_MISSING_TRACKED) + + def test_invalid_identifier(self, magic): + with capture_warnings() as rec: + result = magic._parse_add_magic("--tracked-variable 123bad") + assert result is None + assert_any_warning_contains(rec, MSG_INVALID_IDENTIFIER) + + def test_unknown_argument_raises_systemexit(self, magic): + # argparse still raises SystemExit for unrecognized arguments even with + # exit_on_error=False in some Python versions; SystemExit is not caught + # by the bare `except Exception` in the source. + result = magic._parse_add_magic("--unknown foo") + assert result is None + + +class TestParseLintMagic: + def test_defaults_when_empty(self, magic): + with capture_warnings() as rec: + result = magic._parse_lint_magic("") + assert result == (None, DEFAULT_IP, DEFAULT_PORT, False, False, False) + assert not any(issubclass(w.category, DataLinterWarning) for w in rec) + + def test_positional_and_flags(self, magic): + result = magic._parse_lint_magic("myvar --show-stats --show-na --ip 10.0.0.1 --port 1234") + assert result == ("myvar", "10.0.0.1", 1234, True, True, False) + + def test_show_passing(self, magic): + result = magic._parse_lint_magic("--show-passing") + assert result[4] is False # show_na + assert result[3] is False # show_stats + assert result[5] is True # show_passing + + def test_invalid_ip(self, magic): + with capture_warnings() as rec: + result = magic._parse_lint_magic("--ip not-an-ip") + assert result is None + assert_any_warning_contains(rec, MSG_LINE_PARSE_ERROR) + assert_any_warning_contains(rec, "not valid") + + def test_invalid_port(self, magic): + with capture_warnings() as rec: + result = magic._parse_lint_magic("--port 99999") + assert result is None + assert_any_warning_contains(rec, MSG_LINE_PARSE_ERROR) + + def test_non_integer_port(self, magic): + with capture_warnings() as rec: + result = magic._parse_lint_magic("--port abc") + assert result is None + assert_any_warning_contains(rec, MSG_LINE_PARSE_ERROR) + + +class TestCsvConversion: + def test_dataframe_no_header(self, magic, two_col_frame): + assert magic.dataframe_to_csv_string(two_col_frame, False, ",") == TWO_COL_NO_HEADER + + def test_dataframe_with_header(self, magic, two_col_frame): + assert magic.dataframe_to_csv_string(two_col_frame, True, ",") == TWO_COL_HEADER + + def test_dataframe_custom_delimiter(self, magic, two_col_frame): + assert magic.dataframe_to_csv_string(two_col_frame, True, "|") == TWO_COL_PIPE_HEADER + + def test_ndarray_conversion(self, magic): + arr = np.array([[1, 3], [2, 4]]) + assert magic.ndarray_to_csv_string(arr, False, ",") == TWO_COL_NO_HEADER + assert magic.ndarray_to_csv_string(arr, True, ",") == "x0,x1\n1,3\n2,4\n" + + def test_to_csv_string_dispatches(self, magic, two_col_frame): + assert magic.to_csv_string(two_col_frame, False, ",") == TWO_COL_NO_HEADER + arr = np.array([[10]]) + assert magic.to_csv_string(arr, False, ",") == "10\n" + + def test_unsupported_type_raises(self, magic): + with pytest.raises(TypeError, match="Unsupported type"): + magic.to_csv_string("not a frame", False, ",") + + def test_non_string_columns_generate_names_and_warn(self, magic): + df = pd.DataFrame([[1, 2]], columns=[0, "b"]) + with capture_warnings() as rec: + result = magic.dataframe_to_csv_string(df, True, ",", symbol="mix") + assert result == "x0,x1\n1,2\n" + assert_any_warning_contains(rec, MSG_GENERATED_NAMES) + assert_any_warning_contains(rec, "mix") + + def test_na_representation(self, magic): + df = pd.DataFrame({"a": [1, None]}) + result = magic.dataframe_to_csv_string(df, False, ",") + assert "NA" in result + + +class TestHttpLintRequest: + def test_successful_json_post(self, magic, monkeypatch): + state = _install_http(monkeypatch) + state["response"] = state["FakeResponse"]( + 200, "OK", '{"linting_output": "caf\\u00e9"}' + ) + body = "é".encode("utf-8") + + parsed = magic.http_lint_request( + "localhost", "8080", body, timeout=3.5, path="/v2/lint" + ) + + conn = state["conn"] + assert parsed == {"linting_output": "café"} + assert conn.host == "localhost" + assert conn.port == 8080 + # Connection timeout is always DEFAULT_TIMEOUT, not the argument + assert conn.timeout == DEFAULT_TIMEOUT + assert conn.closed is True + assert conn.requested["method"] == "POST" + assert conn.requested["url"] == "/v2/lint" + assert conn.requested["body"] == body + assert conn.requested["headers"]["Content-Type"] == "application/json" + assert conn.requested["headers"]["Content-Length"] == str(len(body)) + + def test_default_path_and_timeout_constant(self, magic, monkeypatch): + state = _install_http(monkeypatch) + state["response"] = state["FakeResponse"](200, "OK", b'{"ok": true}') + + assert magic.http_lint_request(DEFAULT_IP, DEFAULT_PORT, b"{}") == {"ok": True} + assert state["conn"].timeout == DEFAULT_TIMEOUT + assert state["conn"].requested["url"] == "/api/lint" + + @pytest.mark.parametrize("host", ["0.0.0.0", "::", "[::]"]) + def test_wildcard_hosts_connect_to_loopback(self, magic, monkeypatch, host): + state = _install_http(monkeypatch) + state["response"] = state["FakeResponse"](200, "OK", b"{}") + + assert magic.http_lint_request(host, 9, b"{}") == {} + assert state["conn"].host == DEFAULT_IP + assert state["conn"].port == 9 + + def test_non_200_raises_http_error(self, magic, monkeypatch): + state = _install_http(monkeypatch) + state["response"] = state["FakeResponse"]( + 201, "Created", '{"linting_output": "ignored"}' + ) + + with pytest.raises(LinterHTTPError) as exc: + magic.http_lint_request("127.0.0.1", 10000, b"{}") + + assert exc.value.status == 201 + assert exc.value.reason == "Created" + assert exc.value.body == '{"linting_output": "ignored"}' + assert state["conn"].closed is True + + def test_invalid_json_raises_connection_error(self, magic, monkeypatch): + state = _install_http(monkeypatch) + state["response"] = state["FakeResponse"](200, "OK", b"not json") + + with pytest.raises(LinterConnectionError, match="Invalid JSON"): + magic.http_lint_request("127.0.0.1", 10000, b"{}") + + def test_connection_refused_raises(self, magic, monkeypatch): + state = _install_http(monkeypatch) + state["request_error"] = ConnectionRefusedError("refused") + + with pytest.raises(LinterConnectionError, match="Could not reach"): + magic.http_lint_request("127.0.0.1", 9, b"{}") + + def test_socket_timeout_uses_timeout_argument_in_message(self, magic, monkeypatch): + state = _install_http(monkeypatch) + state["request_error"] = socket.timeout("timed out") + + with pytest.raises(LinterConnectionError, match=r"Timed out after 7s"): + magic.http_lint_request("127.0.0.1", 10000, b"{}", timeout=7) + + +class TestAddMagic: + def test_tracks_dataframe_and_returns_none(self, magic, shell, two_col_frame): + shell.user_ns["df_add"] = two_col_frame + with capture_warnings() as rec: + assert magic.add("--tracked-variable df_add") is None + assert not any(issubclass(w.category, DataLinterWarning) for w in rec) + csv_text, header, delim = magic.all_tracked_variables["df_add"] + assert (csv_text, header, delim) == (TWO_COL_NO_HEADER, False, ",") + + def test_tracks_dataframe_with_header_and_custom_delim(self, magic, shell, two_col_frame): + shell.user_ns["df_add"] = two_col_frame + magic.add("--tracked-variable df_add --data-header --data-delim |") + assert magic.all_tracked_variables["df_add"] == ( + TWO_COL_PIPE_HEADER, + True, + "|", + ) + + def test_tracks_ndarray(self, magic, shell): + shell.user_ns["arr_add"] = np.array([[5, 6]]) + magic.add("--tracked-variable arr_add --data-header") + csv_text, header, delim = magic.all_tracked_variables["arr_add"] + assert header is True + assert "x0,x1" in csv_text or "5,6" in csv_text + + def test_missing_variable_in_namespace_warns(self, magic, shell): + with capture_warnings() as rec: + magic.add("--tracked-variable missing_df") + assert_any_warning_contains(rec, MSG_ADD_FAIL) + assert "missing_df" not in magic.all_tracked_variables + + def test_unsupported_type_warns(self, magic, shell): + shell.user_ns["bad"] = 42 + with capture_warnings() as rec: + magic.add("--tracked-variable bad") + assert_any_warning_contains(rec, MSG_ADD_FAIL) + assert "bad" not in magic.all_tracked_variables + + def test_parse_failure_warns_twice(self, magic): + # _parse_add_magic warns, then add warns again + with capture_warnings() as rec: + magic.add("--tracked-variable 123bad") + msgs = [str(w.message) for w in rec if issubclass(w.category, DataLinterWarning)] + assert any(MSG_INVALID_IDENTIFIER in m for m in msgs) + assert any(MSG_ADD_PARSE_FAIL in m for m in msgs) + + +class TestLintLineMagic: + def test_posts_tracked_frame_and_displays_output(self, magic, shell, monkeypatch, two_col_frame): + displayed = capture_display(monkeypatch) + shell.user_ns["df_line"] = two_col_frame + magic.add("--tracked-variable df_line --data-header --data-delim ;") + displayed.clear() + captured = _capture_lint( + monkeypatch, magic, response={"linting_output": "row 1: ok\nrow 2: ok"} + ) + + with capture_warnings() as rec: + assert magic.lintline("df_line --show-stats --port 2222") is None + + assert not any(issubclass(w.category, DataLinterWarning) for w in rec) + assert captured["ip"] == DEFAULT_IP + assert captured["port"] == 2222 + assert captured["path"] == "/api/lint" + assert_payload_equals( + captured["body"], + make_line_payload( + data="a;b\n1;3\n2;4\n", + delim=";", + header=True, + show_stats=True, + ), + ) + assert_any_display_contains(displayed, MSG_OUTPUT_HEADER) + assert_any_display_contains(displayed, "row 1: ok") + + def test_refreshes_from_user_namespace_before_the_request(self, magic, shell, monkeypatch): + capture_display(monkeypatch) + shell.user_ns["df_refresh"] = pd.DataFrame({"a": [1], "b": [2]}) + magic.add("--tracked-variable df_refresh") + shell.user_ns["df_refresh"] = pd.DataFrame({"a": [7, 8], "b": [2, 9]}) + captured = _capture_lint(monkeypatch, magic) + + magic.lintline("df_refresh --show-na --show-passing") + + assert_payload_equals( + captured["body"], + make_line_payload( + data="7,2\n8,9\n", + show_passing=True, + show_na=True, + ), + ) + assert magic.all_tracked_variables["df_refresh"][0] == "7,2\n8,9\n" + + def test_keeps_cached_csv_when_name_was_deleted(self, magic, shell, monkeypatch): + capture_display(monkeypatch) + shell.user_ns["df_cached"] = pd.DataFrame({"a": [1], "b": [2]}) + magic.add("--tracked-variable df_cached") + del shell.user_ns["df_cached"] + captured = _capture_lint(monkeypatch, magic) + + magic.lintline("df_cached") + + data = json.loads(captured["body"].decode())["linter_input"]["context"]["data"] + assert data == "1,2\n" + + def test_needs_valid_symbol_warns(self, magic, monkeypatch): + capture_display(monkeypatch) + with capture_warnings() as rec: + magic.lintline("unknown_var") + assert_any_warning_contains(rec, MSG_LINTLINE_NEEDS_SYMBOL) + + def test_parse_failure_warns(self, magic, monkeypatch): + capture_display(monkeypatch) + with capture_warnings() as rec: + magic.lintline("--ip badip") + assert_any_warning_contains(rec, MSG_LINTLINE_PARSE_FAIL) + + def test_connection_error_is_displayed_as_warning(self, magic, shell, monkeypatch): + capture_display(monkeypatch) + shell.user_ns["df_err"] = pd.DataFrame({"a": [1]}) + magic.add("--tracked-variable df_err") + + def raise_conn(*a, **k): + raise LinterConnectionError("timeout") + + monkeypatch.setattr(magic, "http_lint_request", raise_conn) + with capture_warnings() as rec: + magic.lintline("df_err") + assert_any_warning_contains(rec, MSG_CANNOT_REACH) + + def test_http_error_is_displayed_as_warning(self, magic, shell, monkeypatch): + capture_display(monkeypatch) + shell.user_ns["df_http"] = pd.DataFrame({"a": [1]}) + magic.add("--tracked-variable df_http") + + def raise_http(*a, **k): + raise LinterHTTPError(400, "Bad Request", "bad data") + + monkeypatch.setattr(magic, "http_lint_request", raise_http) + with capture_warnings() as rec: + magic.lintline("df_http") + assert_any_warning_contains(rec, MSG_HTTP_REJECT) + assert_any_warning_contains(rec, "400") + + +class TestLintCellMagic: + def test_request_uses_every_tracked_variable_and_then_runs_the_cell( + self, magic, shell, monkeypatch + ): + displayed = capture_display(monkeypatch) + shell.user_ns["cell_df"] = pd.DataFrame({"a": [1, 2], "b": [3, 4]}) + shell.user_ns["cell_arr"] = np.array([5]) + magic.add("--tracked-variable cell_df") + magic.add("--tracked-variable cell_arr --data-delim | --data-header") + # Rebind after tracking + shell.user_ns["cell_df"] = pd.DataFrame({"a": [9], "b": [8]}) + displayed.clear() + seen = {} + + def recording_fake(ip, port, body, timeout=DEFAULT_TIMEOUT, path="/api/lint"): + seen["during"] = shell.user_ns.get("cell_flag", "before") + seen["ip"] = ip + seen["port"] = port + seen["body"] = body + return {"linting_output": "clean"} + + monkeypatch.setattr(magic, "http_lint_request", recording_fake) + + result = magic.lintcell( + "--show-stats --show-na --show-passing --port 1234", + "cell_flag = 'after'\ncell_df.iloc[0, 0] = 100\n6 * 7", + ) + + assert seen["during"] == "before" + assert result == 42 + assert shell.user_ns["cell_flag"] == "after" + assert shell.user_ns["cell_df"].iloc[0, 0] == 100 + assert seen["port"] == 1234 + assert seen["ip"] == DEFAULT_IP + + payload = json.loads(seen["body"].decode()) + data = payload["linter_input"]["context"]["data"] + # After refresh, cell_df should be the rebound frame + assert data["cell_df"][0] == "9,8\n" + assert data["cell_df"][1] is False + assert data["cell_df"][2] == "," + # ndarray with header generates x0 + assert "x0" in data["cell_arr"][0] or "5" in data["cell_arr"][0] + assert data["cell_arr"][1] is True + assert data["cell_arr"][2] == "|" + assert payload["linter_input"]["context"]["code"] == ( + "cell_flag = 'after'\ncell_df.iloc[0, 0] = 100\n6 * 7" + ) + assert payload["linter_input"]["options"]["show_stats"] is True + assert_any_display_contains(displayed, "clean") + + def test_deleted_name_keeps_its_cached_csv(self, magic, shell, monkeypatch): + capture_display(monkeypatch) + shell.user_ns["keep_df"] = pd.DataFrame({"a": [1]}) + shell.user_ns["drop_df"] = pd.DataFrame({"b": [2]}) + magic.add("--tracked-variable keep_df") + magic.add("--tracked-variable drop_df --data-header --data-delim ;") + shell.user_ns["keep_df"] = pd.DataFrame({"a": [9]}) + del shell.user_ns["drop_df"] + captured = _capture_lint(monkeypatch, magic) + + assert magic.lintcell("", "1") == 1 + + data = json.loads(captured["body"].decode())["linter_input"]["context"]["data"] + assert data["keep_df"] == ["9\n", False, ","] + assert data["drop_df"] == ["b\n2\n", True, ";"] + assert json.loads(captured["body"].decode())["linter_input"]["context"]["code"] == "1" + + def test_positional_name_is_ignored_and_empty_tracking_still_posts( + self, magic, monkeypatch + ): + capture_display(monkeypatch) + captured = _capture_lint(monkeypatch, magic) + assert magic.lintcell("not_tracked --ip 10.1.1.1 --port 9 --show-passing", "1+1") == 2 + payload = json.loads(captured["body"].decode()) + assert captured["ip"] == "10.1.1.1" + assert captured["port"] == 9 + assert payload["linter_input"]["context"]["data"] == {} + assert payload["linter_input"]["context"]["code"] == "1+1" + assert payload["linter_input"]["options"]["show_passing"] is True + + def test_parse_failure_still_runs_cell(self, magic, shell, monkeypatch): + capture_display(monkeypatch) + with capture_warnings() as rec: + result = magic.lintcell("--ip badip", "3 * 5") + assert result == 15 + assert_any_warning_contains(rec, MSG_LINTCELL_PARSE_FAIL) + + def test_connection_error_still_runs_cell(self, magic, shell, monkeypatch): + capture_display(monkeypatch) + shell.user_ns["err_df"] = pd.DataFrame({"a": [1]}) + magic.add("--tracked-variable err_df") + + def raise_conn(*a, **k): + raise LinterConnectionError("down") + + monkeypatch.setattr(magic, "http_lint_request", raise_conn) + with capture_warnings() as rec: + result = magic.lintcell("", "2 + 2") + assert result == 4 + assert_any_warning_contains(rec, MSG_CANNOT_REACH) + + def test_other_exception_still_runs_cell(self, magic, shell, monkeypatch): + capture_display(monkeypatch) + shell.user_ns["err_df"] = pd.DataFrame({"a": [1]}) + magic.add("--tracked-variable err_df") + + def raise_other(*a, **k): + raise RuntimeError("unexpected") + + monkeypatch.setattr(magic, "http_lint_request", raise_other) + with capture_warnings() as rec: + result = magic.lintcell("", "10") + assert result == 10 + assert_any_warning_contains(rec, MSG_FAILED_READ_CELL)