diff --git a/CHANGELOG.md b/CHANGELOG.md index 3754835..e570d87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,37 @@ ## [Unreleased] +## [0.5.3] - 2026-11-06 + +Security release. Upgrade if your embedding-cache `backend` URL carries a password. + +### Security +- **The embedding-cache backend URL is no longer logged with its credentials.** On every + successful connect, the `embedding_cache.connected` INFO line printed `backend` verbatim, + so a `redis://:@host:port/db` URL put the password in the container log (and in + any log pipeline collecting it). The `FATAL:` stderr message on a failed startup echoed + the same URL. Both now go through `redact_backend_url()`, which keeps scheme, host, port + and db and drops the userinfo, every other query parameter (redis-py accepts + `?password=`) and the fragment. The userinfo ends at the last `@` before the first `?` or + `#`, so a password with an unencoded `/` or `@` is still removed. If any `@` is left after + that (in a query value such as `?password=a@b`, in the fragment, or after a `?`/`#` inside + the password), the URL is ambiguous and is logged as `scheme://`. +- **The `FATAL:` message no longer quotes arbitrary exception text.** redis-py's own errors + (which name only host:port or the socket path) are still shown. Any other exception, such + as urllib's `ValueError` for a bad port, which quotes the port fragment of a malformed URL + and so can carry part of a password, is reduced to its type. The redis-py text is withheld + too, and the message says to percent-encode the password, whenever the URL isn't clean: + when the redacted form has no host (``, or a URL that doesn't start with a valid `scheme://`, which redis-py + still parses for `unix:` and uses the whole remainder, userinfo included, as the socket + path), or when the userinfo has spilled into what redis-py reads as host, port or path + (the text before the URL's last `@` is not the userinfo the URL parser found, as with a + password holding `@` then `/`). +- **certifi:** the lockfile now pins 2026.10.23 (PYSEC-2026-4320, fixed in 2026.10.18). The + image and CI don't install from the lockfile; they resolve the newest certifi at build + time, which already includes the fix. +- If you ran 0.5.2 or earlier with a password in the backend URL, treat that password as + exposed to whatever stores your container logs, and rotate it after upgrading. + ## [0.5.2] - 2026-10-07 Dependency release. Picks up fastapi 0.142.2 for the published image. p2 of diff --git a/pyproject.toml b/pyproject.toml index 994597f..76168db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ollama-queue-proxy" -version = "0.5.2" +version = "0.5.3" description = "Drop-in HTTP proxy for Ollama with priority queuing, per-client auth, and model-aware failover" readme = "README.md" license = { text = "MIT" } diff --git a/src/ollama_queue_proxy/__init__.py b/src/ollama_queue_proxy/__init__.py index 6cc1f6a..3397d50 100644 --- a/src/ollama_queue_proxy/__init__.py +++ b/src/ollama_queue_proxy/__init__.py @@ -13,4 +13,4 @@ # This drifted twice before the check existed: `__version__` stuck at 0.3.1 through # v0.3.2, and at 0.4.0 through v0.5.0, while `main.py` advertised 0.2.0 to every # OpenAPI consumer from v0.3.0 onward without ever being bumped. -__version__ = "0.5.2" +__version__ = "0.5.3" diff --git a/src/ollama_queue_proxy/cache.py b/src/ollama_queue_proxy/cache.py index 4024341..9e9a0ce 100644 --- a/src/ollama_queue_proxy/cache.py +++ b/src/ollama_queue_proxy/cache.py @@ -5,9 +5,12 @@ import hashlib import json import logging +import re import time +from urllib.parse import parse_qs, urlsplit import redis.asyncio as aioredis +from redis.exceptions import RedisError from .config import EmbeddingCacheConfig @@ -25,6 +28,82 @@ errors: dict[str, int] = {} +_SCHEME_RE = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*") + + +def redact_backend_url(url: str) -> str: + """Return the backend URL with credentials removed, for logs and error messages. + + Keeps scheme, host, port and db (path, or the ``db`` query parameter); drops the + userinfo, every other query parameter (redis-py accepts ``?password=``) and the + fragment. Works on the raw string rather than ``urlsplit``, whose netloc ends early + when a password holds an unencoded ``/``: + + - the userinfo boundary is the LAST ``@`` before the first ``?`` or ``#``; + - if an ``@`` is left anywhere after that (in the query, the fragment, or past a + ``?``/``#`` inside the password), the split is ambiguous and the whole location + is replaced with ```` rather than guessing a host. + """ + scheme, sep, rest = url.partition("://") + # A "scheme" that isn't one means the first "://" sits inside the userinfo of a URL + # written without one (redis-py still parses "unix:..."), so it can hold password text. + if not sep or not _SCHEME_RE.fullmatch(scheme): + return "" + cut = min((i for i in (rest.find("?"), rest.find("#")) if i != -1), default=len(rest)) + location, tail = rest[:cut], rest[cut:] + location = location.rpartition("@")[2] + if "@" in location or "@" in tail: + return f"{scheme}://" + query = tail[1:].partition("#")[0] if tail.startswith("?") else "" + db = parse_qs(query).get("db", [""])[0] + suffix = f"?db={db}" if db.isdigit() else "" + return f"{scheme}://{location}{suffix}" + + +def _userinfo_bled(url: str) -> bool: + """True when the URL's userinfo may have spilled into what redis-py parses as host, + port or path: the raw text before the LAST ``@`` after ``://`` is not exactly the + userinfo ``urlsplit`` found (it stops the netloc at the first ``/``, ``?`` or ``#``). + A password holding ``@`` then ``/`` keeps an ``@`` in the netloc, so "is there an + ``@`` in the netloc" is not enough. + """ + scheme_sep, after = url.partition("://")[1:] + if not scheme_sep: + # No "://": redis-py still parses "unix:..." and "unix:/..." and uses the whole + # remainder, userinfo included, as the socket path. Nothing to compare against, + # so any "@" fails closed. + return "@" in url + if "@" not in after: + return False + try: + netloc = urlsplit(url).netloc + except ValueError: + return True + return netloc.rpartition("@")[0] != after.rpartition("@")[0] + + +def _startup_error_detail(url: str, exc: Exception) -> str: + """Describe a startup failure without echoing anything that could be a credential. + + redis-py's own errors name only host:port or the socket path. Anything else, such as + urllib's ``ValueError`` for a bad port, can quote part of a malformed URL, so only + the exception type is given. redis-py's text is withheld too whenever the URL is not + clean: ``redact_backend_url`` could not name a host (unrecognised or ````), + or the userinfo bled into host, port or path (``_userinfo_bled``). Then redis-py's + host:port or socket path can hold password characters. + """ + redacted = redact_backend_url(url) + unclean = redacted == "" or redacted.endswith("://") + if unclean or _userinfo_bled(url): + return ( + f"{type(exc).__name__} (detail withheld: the URL may hold an unencoded " + "credential; percent-encode reserved characters in the password)" + ) + if isinstance(exc, RedisError): + return f"{type(exc).__name__}: {exc}" + return type(exc).__name__ + + def _canonical_json(obj) -> bytes: return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode( "utf-8" @@ -79,11 +158,14 @@ async def startup(self) -> None: socket_connect_timeout=self._cfg.connect_timeout, ) await self._client.ping() - logger.info("embedding_cache.connected backend=%s", self._cfg.backend) + logger.info( + "embedding_cache.connected backend=%s", redact_backend_url(self._cfg.backend) + ) except Exception as e: print( f"FATAL: embedding cache startup failed — could not connect to " - f"'{self._cfg.backend}': {e}. " + f"'{redact_backend_url(self._cfg.backend)}': " + f"{_startup_error_detail(self._cfg.backend, e)}. " "Fix the backend address or set embedding_cache.enabled: false.", file=sys.stderr, ) diff --git a/tests/test_cache.py b/tests/test_cache.py index c055a81..031747d 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -2,15 +2,20 @@ from __future__ import annotations +import logging from unittest.mock import AsyncMock, patch import pytest +from redis.exceptions import ConnectionError as RedisConnectionError from ollama_queue_proxy.cache import ( CACHEABLE_PATHS, EmbeddingCache, _embed_key, _embeddings_key, + _startup_error_detail, + _userinfo_bled, + redact_backend_url, ) from ollama_queue_proxy.config import EmbeddingCacheConfig @@ -117,6 +122,299 @@ async def test_startup_succeeds_on_ping_ok(): await cache.startup() # must not raise +# --------------------------------------------------------------------------- +# Backend URL redaction — the password must never reach a log or stderr +# --------------------------------------------------------------------------- + +SECRET = "s3cret" + + +@pytest.mark.parametrize( + ("url", "expected"), + [ + # password, no username: the shape deployments use + ("redis://:s3cret@valkey:6379/0", "redis://valkey:6379/0"), + ("redis://user:s3cret@valkey:6379/2", "redis://valkey:6379/2"), + ("rediss://default:s3cret@cache.example:6380/0", "rediss://cache.example:6380/0"), + ("unix://:s3cret@/run/valkey.sock?db=3", "unix:///run/valkey.sock?db=3"), + # no credentials: unchanged + ("redis://localhost:6379/0", "redis://localhost:6379/0"), + ("unix:///run/valkey.sock", "unix:///run/valkey.sock"), + # password as a query parameter (redis-py accepts it); other params dropped + ("redis://valkey:6379/0?password=s3cret&db=1", "redis://valkey:6379/0?db=1"), + ("redis://valkey:6379/0#s3cret", "redis://valkey:6379/0"), + # unencoded reserved characters end urlsplit's netloc early; still stripped + ("redis://:s3/cret@valkey:6379/0", "redis://valkey:6379/0"), + ("redis://:s3@cret@valkey:6379/0", "redis://valkey:6379/0"), + ("redis://:s3%40cret@valkey:6379/0", "redis://valkey:6379/0"), + # a '?' or '#' inside the password leaves an '@' past the authority: ambiguous, + # so nothing after :// is kept + ("redis://:s3?cret@valkey:6379/0", "redis://"), + ("redis://:s3#cret@valkey:6379/0", "redis://"), + ("redis://:s3/?cret@valkey:6379/0", "redis://"), + ("redis://u:s3/?c#r%40et@valkey:6379/0?db=1#x", "redis://"), + # an '@' inside a query value or the fragment, with no userinfo at all + ("redis://valkey:6379/0?password=hunt@s3cret", "redis://"), + ("redis://valkey:6379/0?username=u&password=p@s3cret&db=0", "redis://"), + ("unix:///run/valkey.sock?password=a@s3cret&db=2", "unix://"), + ("redis://valkey:6379/0#frag@s3cret", "redis://"), + # not a URL at all: say nothing about it + ("s3cret", ""), + # no real "://", and the password holds one: the text before it is not a scheme + ("unix::s3://cret@/run/x.sock", ""), + ("redis::s3://cret@valkey:6379/0", ""), + ], +) +def test_redact_backend_url(url, expected): + assert redact_backend_url(url) == expected + + +@pytest.mark.asyncio +async def test_connected_log_redacts_password(caplog): + cfg = make_cfg(backend=f"redis://:{SECRET}@valkey:6379/0") + cache = EmbeddingCache(cfg) + + mock_client = AsyncMock() + mock_client.ping.return_value = True + + # Pin the level so the INFO record is produced: the point is that it exists AND is + # redacted, not that it was filtered out. + with ( + caplog.at_level(logging.INFO, logger="ollama_queue_proxy.cache"), + patch("redis.asyncio.from_url", return_value=mock_client) as from_url, + ): + await cache.startup() + + # the real URL, password and all, still goes to the client + assert from_url.call_args.args[0] == f"redis://:{SECRET}@valkey:6379/0" + records = [r for r in caplog.records if "embedding_cache.connected" in r.getMessage()] + assert len(records) == 1 + msg = records[0].getMessage() + assert "valkey:6379/0" in msg + assert SECRET not in msg + assert SECRET not in caplog.text + + +@pytest.mark.asyncio +async def test_startup_failure_stderr_redacts_password(capsys): + cfg = make_cfg(backend=f"redis://:{SECRET}@valkey:6379/0") + cache = EmbeddingCache(cfg) + + mock_client = AsyncMock() + mock_client.ping.side_effect = RedisConnectionError( + "Error 111 connecting to valkey:6379. Connection refused." + ) + + with patch("redis.asyncio.from_url", return_value=mock_client), pytest.raises(SystemExit): + await cache.startup() + + err = capsys.readouterr().err + assert "FATAL" in err + assert "valkey:6379" in err + assert "Connection refused" in err # redis-py's own detail is kept + assert SECRET not in err + + +@pytest.mark.asyncio +async def test_startup_failure_withholds_non_redis_exception_text(capsys): + """A non-redis error (e.g. urllib's port ValueError) can quote the URL: type only.""" + cfg = make_cfg(backend=f"redis://:{SECRET}@valkey:6379/0") + cache = EmbeddingCache(cfg) + + with ( + patch("redis.asyncio.from_url", side_effect=ValueError(f"bad value {SECRET!r}")), + pytest.raises(SystemExit), + ): + await cache.startup() + + err = capsys.readouterr().err + assert "ValueError" in err + assert SECRET not in err + + +@pytest.mark.asyncio +async def test_startup_failure_on_malformed_url_leaks_no_password_fragment(capsys): + """Real redis-py parse of a password with an unencoded '/': urllib's port error + quotes the fragment after the colon. Nothing of the password may reach stderr.""" + cfg = make_cfg(backend="redis://:pw/s3cret@valkey:6379/0") + cache = EmbeddingCache(cfg) + + with pytest.raises(SystemExit): + await cache.startup() + + err = capsys.readouterr().err + assert "redis://valkey:6379/0" in err + assert "pw" not in err.replace("password", "") + assert SECRET not in err + + +@pytest.mark.asyncio +async def test_startup_failure_bled_url_withholds_redis_error_text(capsys): + """When the password bled out of the netloc, even redis-py's host:port text can + carry part of it (here the digits before the '/'), so the detail is withheld.""" + cfg = make_cfg(backend="redis://:4242/s3cret@valkey:6379/0") + cache = EmbeddingCache(cfg) + + mock_client = AsyncMock() + mock_client.ping.side_effect = RedisConnectionError( + "Error 111 connecting to localhost:4242. Connection refused." + ) + + with patch("redis.asyncio.from_url", return_value=mock_client), pytest.raises(SystemExit): + await cache.startup() + + err = capsys.readouterr().err + assert "ConnectionError" in err + assert "4242" not in err + assert SECRET not in err + + +@pytest.mark.asyncio +async def test_startup_failure_password_with_at_and_slash_withholds_detail(capsys): + """A password holding '@' then '/' keeps an '@' in urlsplit's netloc, so a check for + "any '@' in the netloc" misses that redis-py's host:port is now password text.""" + cfg = make_cfg(backend="redis://user:pw@s3cret:1/rest@valkey:6379/0") + cache = EmbeddingCache(cfg) + + mock_client = AsyncMock() + mock_client.ping.side_effect = RedisConnectionError( + "Error 111 connecting to s3cret:1. Connection refused." + ) + + with patch("redis.asyncio.from_url", return_value=mock_client), pytest.raises(SystemExit): + await cache.startup() + + err = capsys.readouterr().err + assert "ConnectionError" in err + assert "redis://valkey:6379/0" in err + assert SECRET not in err + + +@pytest.mark.parametrize( + ("url", "bled"), + [ + ("redis://:s3cret@valkey:6379/0", False), + ("redis://valkey:6379/0", False), + ("unix:///run/valkey.sock", False), + ("unix:/run/valkey.sock", False), # no "//" but no "@": nothing to leak + ("unix::s3cret@/run/x.sock", True), # no "//" and an "@": fail closed + ("unix:s3cret@/run/x.sock", True), + ("redis::s3cret@valkey:6379/0", True), + ("redis://user:pw@s3cret:1/rest@valkey:6379/0", True), + ], +) +def test_userinfo_bled(url, bled): + assert _userinfo_bled(url) is bled + + +def test_startup_error_detail_withholds_text_when_url_is_not_clean(): + """Independently of _userinfo_bled: a URL redact_backend_url can't name a host for + gets no redis-py text, even when nothing bled.""" + exc = RedisConnectionError("Error 2 connecting to /run/valkey.sock. No such file.") + assert not _userinfo_bled("unix:/run/valkey.sock") + assert "withheld" in _startup_error_detail("unix:/run/valkey.sock", exc) + assert "No such file" in _startup_error_detail("unix:///run/valkey.sock", exc) + + +@pytest.mark.parametrize( + "url", + ["unix:s3cret@/run/x.sock", "unix:/:s3cret@/run/x.sock", "unix::s3cret@/run/x.sock?db=0"], +) +@pytest.mark.asyncio +async def test_startup_failure_unix_without_slashes_withholds_detail(url, capsys): + """redis-py parses 'unix:' with no '//' and connects to the whole remainder as the + socket path, so its real ConnectionError quotes the password.""" + from redis.asyncio.connection import parse_url + + path = parse_url(url)["path"] + assert SECRET in path # the premise: redis-py really does put it in the path + cache = EmbeddingCache(make_cfg(backend=url)) + bad_client = AsyncMock() + bad_client.ping.side_effect = RedisConnectionError( + f"Error 2 connecting to {path}. No such file or directory." + ) + with patch("redis.asyncio.from_url", return_value=bad_client), pytest.raises(SystemExit): + await cache.startup() + + err = capsys.readouterr().err + assert "ConnectionError" in err + assert SECRET not in err + + +# --------------------------------------------------------------------------- +# Class probe (SC-54): no fragment of a password reaches either output, for every +# separator in every position redis-py reads a password from, on every scheme. +# --------------------------------------------------------------------------- + +_SEPARATORS = ["", "/", "?", "#", "@", ":", "%40", "/?", "@/", "/@", "?@", "#@", "@?", "@#", "://"] +_HEAD, _TAIL = "HEADPART", "TAILPART" + + +def _probe_urls(): + for sep in _SEPARATORS: + pw = f"{_HEAD}{sep}{_TAIL}" + for scheme in ("redis", "rediss"): + yield f"{scheme}://:{pw}@valkey:6379/0" + yield f"{scheme}://user:{pw}@valkey:6379/0" + yield f"{scheme}://valkey:6379/0?password={pw}" + yield f"{scheme}://valkey:6379/0?password={pw}&db=1" + yield f"{scheme}://valkey:6379/0#{pw}" + yield f"unix://:{pw}@/run/valkey.sock?db=1" + yield f"unix:///run/valkey.sock?password={pw}" + # no "//": redis-py still parses these, and a unix: one puts the whole remainder, + # userinfo included, into the socket path + for scheme, where in ( + ("redis", "valkey:6379/0"), + ("rediss", "valkey:6379/0"), + ("unix", "/run/valkey.sock"), + ): + for slash in ("", "/"): + yield f"{scheme}:{slash}:{pw}@{where}" + yield f"{scheme}:{slash}{pw}@{where}" + yield f"{scheme}:{slash}user:{pw}@{where}?db=0" + + +def _redis_py_failure(url: str) -> Exception: + """The exception redis-py would raise connecting to this URL: its real parse, and its + real error text shape (format_error_message: host:port or the socket path).""" + from redis.asyncio.connection import parse_url + + try: + kw = parse_url(url) + except ValueError as e: # urllib's port error and redis-py's own refusals + return e + where = kw.get("path") or f"{kw.get('host', 'localhost')}:{kw.get('port', 6379)}" + return RedisConnectionError(f"Error 111 connecting to {where}. Connection refused.") + + +@pytest.mark.parametrize("url", list(_probe_urls())) +def test_probe_redact_backend_url_leaks_no_password_fragment(url): + out = redact_backend_url(url) + assert _HEAD not in out and _TAIL not in out, out + + +@pytest.mark.parametrize("url", list(_probe_urls())) +@pytest.mark.asyncio +async def test_probe_startup_outputs_leak_no_password_fragment(url, caplog, capsys): + cache = EmbeddingCache(make_cfg(backend=url)) + ok_client = AsyncMock() + ok_client.ping.return_value = True + with ( + caplog.at_level(logging.INFO, logger="ollama_queue_proxy.cache"), + patch("redis.asyncio.from_url", return_value=ok_client), + ): + await cache.startup() + + failing = EmbeddingCache(make_cfg(backend=url)) + bad_client = AsyncMock() + bad_client.ping.side_effect = _redis_py_failure(url) + with patch("redis.asyncio.from_url", return_value=bad_client), pytest.raises(SystemExit): + await failing.startup() + + for text in (caplog.text, capsys.readouterr().err): + assert _HEAD not in text and _TAIL not in text, text + + # --------------------------------------------------------------------------- # EmbeddingCache — cache miss / hit # --------------------------------------------------------------------------- diff --git a/uv.lock b/uv.lock index a002d24..8dce5c7 100644 --- a/uv.lock +++ b/uv.lock @@ -58,11 +58,11 @@ wheels = [ [[package]] name = "certifi" -version = "2026.7.22" +version = "2026.10.23" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, ] [[package]] @@ -357,7 +357,7 @@ wheels = [ [[package]] name = "ollama-queue-proxy" -version = "0.5.2" +version = "0.5.3" source = { editable = "." } dependencies = [ { name = "fastapi" },