diff --git a/generator/captcha_solver.py b/generator/captcha_solver.py index c1d0603..f60f623 100644 --- a/generator/captcha_solver.py +++ b/generator/captcha_solver.py @@ -225,11 +225,12 @@ def _need_ocr(self): Returns None (and logs) when ddddocr is missing instead of raising out of calculate(). """ - if self.ocr is None and self._ocr_factory is not None: - self.ocr = self._ocr_factory() if self.ocr is None: try: - return self._get_ocr() + # the factory IS _get_ocr in production: its RuntimeError + # must not escape only because it arrived via the factory + self.ocr = (self._ocr_factory() if self._ocr_factory is not None + else self._get_ocr()) except RuntimeError as e: self.logger.warning(str(e)) return None diff --git a/generator/claim500.py b/generator/claim500.py index ce21c9a..8e2ea32 100644 --- a/generator/claim500.py +++ b/generator/claim500.py @@ -31,6 +31,7 @@ from invite_claim import ( # noqa: E402 _VERIFY, + client_timezone, device_login, per_account_device_mid, _proxies, @@ -51,7 +52,6 @@ 'http-referer': 'https://zcode.z.ai', 'user-agent': 'ZCode/3.18.0', 'x-client-language': 'en-US', - 'x-client-timezone': 'Asia/Tehran', 'x-os-category': 'windows', 'x-os-version': 'Windows 11 Pro', 'x-platform': 'win32-x64', @@ -111,6 +111,7 @@ def _headers(zjwt: str) -> dict: h = dict(CLAIM_HEADERS_BASE) h['authorization'] = 'Bearer ' + zjwt h['x-device-mid'] = per_account_device_mid(zjwt) + h['x-client-timezone'] = client_timezone() h['x-request-id'] = str(__import__('uuid').uuid4()) return h diff --git a/generator/export_starter.py b/generator/export_starter.py index 405ca58..e495a47 100644 --- a/generator/export_starter.py +++ b/generator/export_starter.py @@ -20,6 +20,7 @@ import sys from plan_check import get_zcode_jwt, get_plan, is_start_plan, DEVICE_MID +import plan_check HERE = os.path.dirname(os.path.abspath(__file__)) # destination for exported snapshots; override per machine (or copy the @@ -99,48 +100,54 @@ def main(): print(f"candidate accounts: {len(accounts)}") os.makedirs(GATEWAY_ACCOUNTS, exist_ok=True) exported = 0 + failed = 0 for email, chat_token in accounts: print(f"\n=== {email} ===") - zjwt = get_zcode_jwt(chat_token, log=lambda *a: print(' ', *a)) - if not zjwt: - print(" no zcodeJWT — skip") - continue - plan = get_plan(zjwt) - if not plan: - print(" no plan data — skip") - continue - starter = plan.get('starter') - referral = plan.get('referral') - print(f" plans={plan.get('plans')} starter={starter} referral={referral}") - if not (starter or referral or export_all): - continue - payload = jwt_payload(zjwt) - ident = payload.get('user_id') or payload.get('sub') or email - plans_list = plan.get('plans') or [] - chosen_plan = next((p for p in plans_list if is_start_plan(p)), - plans_list[0] if plans_list else '') - fields = { - 'captured_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), - 'source': 'ZCODEAUTO chat.z.ai signup (OAuth CLI)', - 'account_id': ident, - 'email': email, - 'name': '', - 'device_mid': DEVICE_MID, - 'plan': chosen_plan, - 'access_token': zjwt, - } - if payload.get('exp'): - fields['expires_at'] = str(payload['exp']) - fields['expires_at_ms'] = str(int(payload['exp']) * 1000) - else: - fields['expires_at_ms'] = '0' - # drop empty values so the file matches ZcodeGate's own output exactly - fields = {k: v for k, v in fields.items() if v != ''} - dest = os.path.join(GATEWAY_ACCOUNTS, safe_name(email)) - write_snapshot(dest, fields) - exported += 1 - print(f" -> wrote {dest}") - print(f"\nexported {exported} account(s) to {GATEWAY_ACCOUNTS}") + try: + zjwt = get_zcode_jwt(chat_token, log=lambda *a: print(' ', *a)) + if not zjwt: + print(" no zcodeJWT — skip") + continue + plan = get_plan(zjwt) + if not plan: + print(" no plan data — skip") + continue + starter = plan.get('starter') + referral = plan.get('referral') + print(f" plans={plan.get('plans')} starter={starter} referral={referral}") + if not (starter or referral or export_all): + continue + payload = jwt_payload(zjwt) + ident = payload.get('user_id') or payload.get('sub') or email + plans_list = plan.get('plans') or [] + chosen_plan = next((p for p in plans_list if is_start_plan(p)), + plans_list[0] if plans_list else '') + fields = { + 'captured_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'source': 'ZCODEAUTO chat.z.ai signup (OAuth CLI)', + 'account_id': ident, + 'email': email, + 'name': '', + 'device_mid': plan_check.per_account_device_mid(zjwt) or DEVICE_MID, + 'plan': chosen_plan, + 'access_token': zjwt, + } + if payload.get('exp'): + fields['expires_at'] = str(payload['exp']) + fields['expires_at_ms'] = str(int(payload['exp']) * 1000) + else: + fields['expires_at_ms'] = '0' + # drop empty values so the file matches ZcodeGate's own output exactly + fields = {k: v for k, v in fields.items() if v != ''} + dest = os.path.join(GATEWAY_ACCOUNTS, safe_name(email)) + write_snapshot(dest, fields) + exported += 1 + print(f" -> wrote {dest}") + except Exception as e: + failed += 1 + print(f" failed ({type(e).__name__}: {e}) — skip") + print(f"\nexported {exported} account(s) to {GATEWAY_ACCOUNTS}" + + (f" ({failed} failed)" if failed else "")) if __name__ == '__main__': diff --git a/generator/invite_claim.py b/generator/invite_claim.py index a6c4324..8f255ff 100644 --- a/generator/invite_claim.py +++ b/generator/invite_claim.py @@ -25,7 +25,7 @@ sys.stderr.reconfigure(encoding='utf-8', errors='replace') from utils import BROWSER_ARGS -from plan_check import APP_VERSION, _request, direct_requests +from plan_check import APP_VERSION, _request, client_timezone, direct_requests # NOTE: playwright is imported lazily inside claim_invite(). The referral # helpers below (referral_summary / send_one_message) are imported by the @@ -502,6 +502,27 @@ def _proxies(): return {'http': raw, 'https': raw} +def _pw_proxy(proxy_url): + """Playwright proxy dict for `proxy_url`, or None for a direct connection. + + A portless value (socks5://1.2.3.4) would otherwise become + "socks5://1.2.3.4:None" and Chromium refuses to launch — same guard as + main.pw_proxy(). Credentials travel as separate fields: Chromium drops + them from 'server' alone and then shows an auth popup. + """ + if not proxy_url: + return None + from urllib.parse import urlparse + u = urlparse(proxy_url) + if not u.hostname or not u.port: + return None + d = {'server': f"{u.scheme or 'http'}://{u.hostname}:{u.port}"} + if u.username: + d['username'] = u.username + d['password'] = u.password or '' + return d + + def _direct_proxy_while(fn): """Run `fn` with direct mode when force_direct=True; always restore.""" import functools @@ -539,7 +560,6 @@ def per_account_device_mid(token: str) -> str: 'http-referer': 'https://zcode.z.ai', 'user-agent': 'ZCode/' + APP_VERSION, 'x-client-language': 'en-US', - 'x-client-timezone': 'Asia/Tehran', 'x-device-mid': '58aca3b3-3239-4cb7-9c70-df3e5488fb99', 'x-os-category': 'windows', 'x-os-version': '10.0.26200', @@ -577,12 +597,15 @@ def report_login_device(zjwt: str, user_id: str = None, device_mid: str = None, mid = device_mid or per_account_device_mid(zjwt) h = dict(APP_HEADERS) h['authorization'] = 'Bearer ' + zjwt + h['x-client-timezone'] = client_timezone() + if mid: + h['x-device-mid'] = mid endpoint = 'https://zcode.z.ai/api/v1/event/report' # 1. app_login_ck body_ck = { 'event_id': str(_uuid.uuid4()), - 'client_timezone': 'Asia/Tehran', + 'client_timezone': client_timezone(), 'client_language': 'en-US', 'element_name': 'app_login_ck', 'event_region': 'app', @@ -608,7 +631,7 @@ def report_login_device(zjwt: str, user_id: str = None, device_mid: str = None, # 2. app_login_success (the critical event that credits login_device) body_success = { 'event_id': str(_uuid.uuid4()), - 'client_timezone': 'Asia/Tehran', + 'client_timezone': client_timezone(), 'client_language': 'en-US', 'element_name': 'app_login_success', 'event_region': 'app_profile', @@ -667,6 +690,10 @@ def device_login(chat_token, log=print, force_direct: bool = False): poll_token = secrets.token_hex(32) h = dict(APP_HEADERS) h['authorization'] = 'Bearer ' + poll_token + h['x-client-timezone'] = client_timezone() + mid = per_account_device_mid(chat_token) + if mid: + h['x-device-mid'] = mid # zcode.z.ai answers 502 in bursts; init is idempotent (it just opens a # new flow), so a few quick retries beat aborting the whole login r = None @@ -848,19 +875,13 @@ def claim_invite(chat_token: str, invite_code: str = 'MT9GNDQ8', log=print, viewport={'width': 1280, 'height': 820}, user_agent=UA, locale='en-US', timezone_id='America/New_York') _px = _proxies() - if _px and _px.get('https'): - # embedded credentials (http://user:pass@host:port) must be split - # into separate fields — Chromium drops them from 'server' alone - # and then shows an auth popup (net::ERR_INVALID_AUTH_CREDENTIALS) - from urllib.parse import urlparse - _u = urlparse(_px['https']) - _pd = {'server': f"{_u.scheme or 'http'}://{_u.hostname}:{_u.port}"} - if _u.username: - _pd['username'] = _u.username - _pd['password'] = _u.password or '' + _pd = _pw_proxy((_px or {}).get('https')) + if _pd: launch_kwargs['proxy'] = _pd elif force_direct: log(' invite: direct mode — claiming on the real IP') + elif _px and _px.get('https'): + log(' invite: proxy has no host:port — launching direct') import fingerprint as _fp _px_url = (_px or {}).get('https') diff --git a/generator/plan_check.py b/generator/plan_check.py index 4f2fe08..b2d7710 100644 --- a/generator/plan_check.py +++ b/generator/plan_check.py @@ -496,7 +496,7 @@ def _plan_events(zjwt): pass mid = per_account_device_mid(zjwt) base = { - "client_timezone": "Asia/Tehran", + "client_timezone": client_timezone(), "client_language": "en-US", "event_text": "", "user_id": jwt_user_id(zjwt), diff --git a/generator/proxy_parse.py b/generator/proxy_parse.py index dd4d283..7fbf994 100644 --- a/generator/proxy_parse.py +++ b/generator/proxy_parse.py @@ -80,8 +80,13 @@ def _is_private_octets(parts: list[int]) -> bool: def valid_endpoint(host: str, port: int) -> bool: if not (0 < port < 65536): return False + octets = host.split(".") + # getaddrinfo/inet_aton read a leading zero as octal ("0177" == 127), so a + # dotted quad like 0177.0.0.1 would slip past the loopback check below + if any(len(o) > 1 and o.startswith("0") for o in octets): + return False try: - parts = [int(o) for o in host.split(".")] + parts = [int(o) for o in octets] except ValueError: return False if len(parts) != 4 or any(p > 255 for p in parts): diff --git a/generator/recheck_plans.py b/generator/recheck_plans.py index ea97b8d..aa589c4 100644 --- a/generator/recheck_plans.py +++ b/generator/recheck_plans.py @@ -96,42 +96,51 @@ def main(): log = lambda m: print(m, flush=True) got = 0 + failed = 0 for n, (email, _password, token) in enumerate(rows, 1): log(f"[{n}/{len(rows)}] {email}") - zjwt = get_zcode_jwt(token, log=lambda *a, **k: None) - if not zjwt: - log(' login failed') - record = {'email': email, 'plans': None, 'error': 'login'} - else: - plan = get_plan(zjwt, log=lambda *a, **k: None) or {} - ids = plan.get('plans') or [] - if not ids and escalate: - log(' no plan — escalating to the full desktop device_login') - try: - from invite_claim import device_login - zjwt2, _ = device_login(token, log=lambda *a, **k: None) - if zjwt2: - zjwt = zjwt2 - time.sleep(2) - plan = get_plan(zjwt, log=lambda *a, **k: None) or {} - ids = plan.get('plans') or [] - except Exception as e: - log(f' escalate failed ({type(e).__name__}: {e})') - has_start = any(is_start_plan(p) for p in ids) - got += 1 if has_start else 0 - record = {'email': email, 'plans': ids, 'starter': has_start, - 'referral': plan.get('referral'), - 'campaign': plan.get('campaign'), - 'entitlements': plan.get('entitlements'), - 'ts': int(time.time())} - if has_start: - with open(startplan_path, 'a', encoding='utf-8') as f: - f.write(f"{email}:{_password}:{token}\n") - log(f" -> {ids}") + try: + zjwt = get_zcode_jwt(token, log=lambda *a, **k: None) + if not zjwt: + log(' login failed') + record = {'email': email, 'plans': None, 'error': 'login'} + else: + plan = get_plan(zjwt, log=lambda *a, **k: None) or {} + ids = plan.get('plans') or [] + if not ids and escalate: + log(' no plan — escalating to the full desktop device_login') + try: + from invite_claim import device_login + zjwt2, _ = device_login(token, log=lambda *a, **k: None) + if zjwt2: + zjwt = zjwt2 + time.sleep(2) + plan = get_plan(zjwt, log=lambda *a, **k: None) or {} + ids = plan.get('plans') or [] + except Exception as e: + log(f' escalate failed ({type(e).__name__}: {e})') + has_start = any(is_start_plan(p) for p in ids) + got += 1 if has_start else 0 + record = {'email': email, 'plans': ids, 'starter': has_start, + 'referral': plan.get('referral'), + 'campaign': plan.get('campaign'), + 'entitlements': plan.get('entitlements'), + 'ts': int(time.time())} + if has_start: + with open(startplan_path, 'a', encoding='utf-8') as f: + f.write(f"{email}:{_password}:{token}\n") + log(f" -> {ids}") + except Exception as e: + failed += 1 + log(f' failed ({type(e).__name__}: {e})') + record = {'email': email, 'plans': None, + 'error': f'{type(e).__name__}: {e}'} with open(plans_path, 'a', encoding='utf-8') as f: f.write(json.dumps(record, ensure_ascii=False) + "\n") log(f"\n{got}/{len(rows)} accounts currently hold a Start Plan") + if failed: + log(f"{failed} account(s) failed — see the error field in {plans_path}") log(f"ledger: {plans_path}") return 0 diff --git a/scripts/smoke/live.py b/scripts/smoke/live.py index 3958be2..f60acd6 100644 --- a/scripts/smoke/live.py +++ b/scripts/smoke/live.py @@ -34,6 +34,7 @@ async def run(args): if args.infer: with tempfile.TemporaryDirectory() as folder: store = Store(str(Path(folder) / 'smoke.db')) + service = None try: service = ChatService(cfg, pool, tokens, client, Registry(), store, JsonlCapture(folder, False)) model = 'GLM-5.3-Flash' @@ -43,6 +44,11 @@ async def run(args): message = json.loads(result.raw) print('Live inference OK:', message['stop_reason'], json.dumps(message['usage'])) finally: + if service is not None: + # complete() returns before the usage record lands in + # the store's executor; draining it first avoids a + # silent write-after-close + await service.aclose() store.close() return 0 except Exception as exc: diff --git a/tests/test_api_bugfix_round.py b/tests/test_api_bugfix_round.py new file mode 100644 index 0000000..af78c41 --- /dev/null +++ b/tests/test_api_bugfix_round.py @@ -0,0 +1,328 @@ +"""Regression tests for the second review-bug round (API/health/service/generator).""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import threading +import time +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest +import yaml +from fastapi import HTTPException +from fastapi.testclient import TestClient +from zcode_gateway import api_admin, api_dash +from zcode_gateway.app import create_app +from zcode_gateway.claim_schedule import ClaimScheduler +from zcode_gateway.config import Config +from zcode_gateway.generator import GeneratorManager, ProxyChecker +from zcode_gateway.pool import Account, AccountState, PoolManager +from zcode_gateway.service import ChatService +from zcode_gateway.store import JsonlCapture + + +def _cfg(tmp_path): + return Config( + server={'host': '127.0.0.1', 'admin_key': 'dummy-admin', + 'client_keys': [{'key': 'fixture-client-key', 'name': 'fixture'}]}, + accounts={'source': 'accounts_dir', 'dir': str(tmp_path / 'accounts')}, + store={'sqlite_path': str(tmp_path / 'usage.db')}, + update={'enabled': False}) + + +def test_health_tolerates_non_ascii_update_probe(tmp_path, monkeypatch): + monkeypatch.setenv('ZCODE_GATEWAY_UPDATE_NONCE', 'a' * 32) + cfg = _cfg(tmp_path) + cfg.server.host = '0.0.0.0' + cfg.server.expose_ops = False + with TestClient(create_app(cfg), client=('198.51.100.3', 12345), + base_url='http://127.0.0.1:8788') as client: + assert client.get('/health').status_code == 404 + malformed = client.get('/health', headers={b'X-AutoZCoder-Update': b'\xe9'}) + assert malformed.status_code == 404 + probe = client.get('/health', headers={'X-AutoZCoder-Update': 'a' * 32}) + assert probe.status_code == 200 + + +@pytest.mark.asyncio +async def test_locate_tears_down_scheduler_and_old_manager(tmp_path, monkeypatch): + gen = tmp_path / 'gen' + gen.mkdir() + (gen / 'main.py').write_text('') + (gen / 'config.py').write_text('') + old = GeneratorManager(gen, data_dir=tmp_path / 'data') + scheduler = ClaimScheduler(run=lambda: None, enabled=True, + state_path=tmp_path / 'schedule.json', + poll_seconds=1.0) + scheduler.start() + old.claim_scheduler = scheduler + + cfg = Config() + cfg._config_file = tmp_path / 'config.yaml' + cfg._config_file.write_text(yaml.safe_dump(cfg.model_dump(mode='json'))) + state = SimpleNamespace(cfg=cfg, generator=old, claim_scheduler=scheduler) + monkeypatch.setattr(api_dash, 'get_state', lambda request: state) + + result = await api_dash.generator_locate_set(None, {'path': str(gen)}, 'admin') + + assert result['ok'] is True + assert state.generator is None and state.claim_scheduler is None + assert scheduler._thread is not None and not scheduler._thread.is_alive() + assert old._closed.is_set() + + +@pytest.mark.asyncio +async def test_blocking_generator_and_claim_endpoints_run_off_event_loop(monkeypatch): + main = threading.current_thread() + calls = [] + + class Manager: + def stop_claim(self, name): + calls.append(('stop_claim', threading.current_thread() is main)) + return True + + def remove_claim(self, name): + calls.append(('remove_claim', threading.current_thread() is main)) + return True + + def stop_worker(self, name): + calls.append(('stop_worker', threading.current_thread() is main)) + return True + + def stop_all(self): + calls.append(('stop_all', threading.current_thread() is main)) + + def remove_worker(self, name): + calls.append(('remove_worker', threading.current_thread() is main)) + return True + + class Checker: + def stop(self): + calls.append(('checker.stop', threading.current_thread() is main)) + + def status(self): + calls.append(('checker.status', threading.current_thread() is main)) + return {'state': 'idle'} + + def run(self, action, **_kwargs): + calls.append((f'checker.run:{action}', threading.current_thread() is main)) + return {'action': action} + + mgr = Manager() + mgr.checker = Checker() + monkeypatch.setattr(api_dash, '_generator', lambda request: mgr) + + assert await api_admin.claim_stop(None, {'name': 'c1'}, 'admin') == {'stopped': 'c1'} + assert await api_admin.claim_remove(None, {'name': 'c1'}, 'admin') == {'removed': 'c1'} + assert await api_dash.generator_stop(None, {'name': 'w1'}, 'admin') == {'stopped': True} + assert await api_dash.generator_stopall(None, 'admin') == {'ok': True} + assert await api_dash.generator_worker_remove(None, {'name': 'w1'}, 'admin') == {'removed': 'w1'} + assert await api_dash.generator_proxies_stop(None, 'admin') == {'ok': True} + assert await api_dash.generator_proxies(None, 'admin') == {'state': 'idle'} + assert await api_dash.generator_proxies_run(None, {'action': 'recheck'}, 'admin') == {'action': 'recheck'} + assert await api_dash.generator_proxies_maintain(None, {'action': 'prune'}, 'admin') == {'action': 'prune'} + assert await api_dash.generator_proxies_countries(None, 'admin') == {'action': 'countries'} + + assert [name for name, _ in calls] == [ + 'stop_claim', 'remove_claim', 'stop_worker', 'stop_all', 'remove_worker', + 'checker.stop', 'checker.status', 'checker.run:recheck', + 'checker.run:prune', 'checker.run:countries'] + assert not any(ran_on_loop for _, ran_on_loop in calls) + + +def test_record_skips_deepcopy_when_capture_is_disabled(tmp_path): + class Body(dict): + def __deepcopy__(self, memo): + raise AssertionError('capture disabled: body must not be deep-copied') + + records = [] + capture = JsonlCapture(str(tmp_path / 'captures'), enabled=False) + service = ChatService(Config(), None, None, None, None, + SimpleNamespace(record=lambda **rec: records.append(rec)), + capture) + service._record(Account(id='dummy'), {'x': 'y'}, Body(stream=True), 200, {}, 'resp', 1.0, + 'key', 'openai', 'GLM-5.3-Flash', {'input_tokens': 2, 'output_tokens': 3}) + + assert len(records) == 1 + assert records[0]['total_tokens'] == 5 and records[0]['status'] == 200 + assert not capture.path.exists() or capture.path.read_text(encoding='utf-8') == '' + + enabled_capture = JsonlCapture(str(tmp_path / 'captures-on'), enabled=True) + service = ChatService(Config(), None, None, None, None, + SimpleNamespace(record=lambda **rec: records.append(rec)), + enabled_capture) + service._record(Account(id='dummy'), {'x': 'y'}, {'stream': True}, 200, {}, 'resp', 1.0, + 'key', 'openai', 'GLM-5.3-Flash', {'input_tokens': 1}) + lines = enabled_capture.path.read_text(encoding='utf-8').splitlines() + assert len(lines) == 1 and '"account_id": "dummy"' in lines[0] + + +@pytest.mark.asyncio +async def test_update_idle_check_snapshots_manager_maps_under_lock(): + lock = threading.Lock() + observed = [] + + class LockedDict(dict): + def values(self): + observed.append(lock.locked()) + return super().values() + + mgr = SimpleNamespace( + auto_generate=SimpleNamespace(status=lambda: {'status': 'idle'}), + _claim_all_running=False, + workers=LockedDict({'w': SimpleNamespace(done=threading.Event())}), + claims=LockedDict(), + checker=SimpleNamespace(alive=lambda: False), + _lock=lock) + state = SimpleNamespace(pool=SimpleNamespace( + snapshot=AsyncMock(return_value={'detail': []})), generator=mgr) + + with pytest.raises(HTTPException) as error: + await api_dash._check_update_idle(state) + assert error.value.status_code == 409 + + mgr.workers.clear() + await api_dash._check_update_idle(state) + + assert observed and all(observed) + + +@pytest.mark.asyncio +async def test_proxy_auto_verify_validates_interval(tmp_path, monkeypatch): + checker = ProxyChecker(tmp_path) + monkeypatch.setattr(api_dash, '_generator', + lambda request: SimpleNamespace(checker=checker)) + + with pytest.raises(HTTPException) as error: + await api_admin.proxy_auto_verify(None, {'enabled': True, 'interval_min': 'abc'}, 'admin') + assert error.value.status_code == 400 + assert 'interval_min' in error.value.detail + assert checker.auto_verify is False + + assert await api_admin.proxy_auto_verify( + None, {'enabled': True, 'interval_min': '12'}, 'admin') == { + 'auto_verify': True, 'interval_min': 12} + assert await api_admin.proxy_auto_verify( + None, {'enabled': True, 'interval_min': 9}, 'admin') == { + 'auto_verify': True, 'interval_min': 9} + assert await api_admin.proxy_auto_verify( + None, {'enabled': True, 'interval_min': ''}, 'admin') == { + 'auto_verify': True, 'interval_min': 9} + await api_admin.proxy_auto_verify(None, {'enabled': False}, 'admin') + assert checker.auto_verify is False + + +def test_pool_has_candidates_tolerates_non_ascii_bytes(tmp_path): + gen = tmp_path / 'gen' + gen.mkdir() + mgr = GeneratorManager(gen, data_dir=tmp_path / 'data') + + # cp1252 (Windows default) and UTF-8 both reject these bytes; errors="ignore" + # must keep the scan alive instead of escaping as UnicodeDecodeError. + mgr.pool_file.write_bytes(b'# \x81\x8d unreadable comment\nsocks5://127.0.0.1:1080\n') + assert mgr._pool_has_candidates() is True + + mgr.pool_file.write_bytes(b'# \x81\x8d no candidates here\n') + assert mgr._pool_has_candidates() is False + + +@pytest.mark.asyncio +async def test_admin_readd_revives_parked_accounts(monkeypatch): + dead = Account(id='dead-1', email='dead@example.test', access_token='old', + source='admin', state=AccountState.DEAD) + dead.death_reason = 'token revoked' + dead.auth_strikes = 3 + drained = Account(id='drained-1', email='drained@example.test', access_token='old', + source='admin', state=AccountState.EXHAUSTED, quota_strikes=2) + pool = PoolManager(Config().pool, [dead, drained]) + state = SimpleNamespace(cfg=Config(), pool=pool) + monkeypatch.setattr(api_admin, 'get_state', lambda request: state) + + await api_admin.add_account(None, {'access_token': 'fresh-dead', 'id': 'dead-1', + 'email': 'dead@example.test'}, 'admin') + revived = await pool.find('dead-1') + assert revived.state is AccountState.READY and revived.death_reason == '' + assert revived.access_token == 'fresh-dead' and revived.auth_strikes == 0 + + await api_admin.add_account(None, {'access_token': 'fresh-drained', 'id': 'drained-1', + 'email': 'drained@example.test'}, 'admin') + revived = await pool.find('drained-1') + assert revived.state is AccountState.READY and revived.quota_strikes == 0 + assert revived.access_token == 'fresh-drained' + await pool.aclose() + + +@pytest.mark.asyncio +async def test_device_login_revives_parked_account(tmp_path, monkeypatch): + from zcode_gateway import device_auth + + canonical = Account(id='same', access_token='old', state=AccountState.DEAD) + canonical.death_reason = 'auth strikes' + pool = PoolManager(Config().pool, [canonical]) + cfg = Config(accounts={'source': 'accounts_dir', 'dir': str(tmp_path)}) + checked = [] + state = SimpleNamespace( + cfg=cfg, pool=pool, + tokens=SimpleNamespace(check_account=AsyncMock( + side_effect=lambda account: checked.append(account)))) + monkeypatch.setattr(api_dash, 'get_state', lambda _: state) + gate = asyncio.Event() + created = [] + + class Login: + def __init__(self, *args, **kwargs): + self.state = 'starting' + self._ready = asyncio.Event() + self._ready.set() + self.callback = kwargs['on_account'] + created.append(self) + + async def run(self): + await gate.wait() + + def status(self): + return {'state': self.state} + + monkeypatch.setattr(device_auth, 'DeviceLogin', Login) + await api_dash.login_start(None, 'admin') + await created[0].callback(Account(id='same', access_token='new')) + assert canonical.state is AccountState.READY and canonical.death_reason == '' + assert canonical.access_token == 'new' and checked == [canonical] + gate.set() + await created[0]._task + await pool.aclose() + + +@pytest.mark.asyncio +async def test_jwt_import_revives_parked_account(tmp_path, monkeypatch): + def part(value): + return base64.urlsafe_b64encode(json.dumps(value).encode()).decode().rstrip('=') + + ident = '99aa3247-908c-433c-874f-616aa15d6e02' + token = (part({'alg': 'HS256'}) + '.' + + part({'sub': ident, 'exp': int(time.time()) + 3600}) + '.sig') + cfg = Config(accounts={'source': 'accounts_dir', 'dir': str(tmp_path / 'accounts')}) + existing = Account(id=ident, access_token='old', state=AccountState.DEAD, + auth_strikes=3, quota_strikes=2) + pool = PoolManager(cfg.pool, [existing]) + lock = asyncio.Lock() + state = SimpleNamespace(cfg=cfg, pool=pool, + tokens=SimpleNamespace(_lock_for=lambda _: lock, _scan_at=1)) + monkeypatch.setattr(api_admin, 'get_state', lambda request: state) + original = httpx.AsyncClient + transport = httpx.MockTransport(lambda request: httpx.Response( + 200, json={'code': 0, 'data': {'plans': [], 'balances': []}})) + monkeypatch.setattr(api_admin.httpx, 'AsyncClient', + lambda **kwargs: original(**kwargs, transport=transport)) + + await api_admin.import_jwt_account(None, {'token': token}, 'admin') + + revived = await pool.find(ident) + assert revived.state is AccountState.READY + assert revived.access_token == token + assert revived.auth_strikes == 0 and revived.quota_strikes == 0 + await pool.aclose() diff --git a/tests/test_generator_bugfixes.py b/tests/test_generator_bugfixes.py new file mode 100644 index 0000000..c56dd2e --- /dev/null +++ b/tests/test_generator_bugfixes.py @@ -0,0 +1,334 @@ +"""Generator bug-sweep regressions. + +Each test pins one behaviour a review found broken: the lazy ddddocr factory +escaping calculate(), one shared device id / timezone across every account, +one flaky account aborting a whole batch, a portless proxy crashing Playwright, +and leading-zero octets bypassing the loopback guard. +""" +from __future__ import annotations + +import base64 +import importlib.util +import json +import logging +import sys +import types +from pathlib import Path +from types import SimpleNamespace + +import pytest + +GEN = Path(__file__).resolve().parent.parent / "generator" +if str(GEN) not in sys.path: + sys.path.insert(0, str(GEN)) + + +def _module(name): + try: + return __import__(name) + except ImportError: + return None + + +def _load_script(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + try: + spec.loader.exec_module(module) + finally: + sys.modules.pop(name, None) + return module + + +def _jwt(user_id="u1"): + payload = base64.urlsafe_b64encode( + json.dumps({"user_id": user_id}).encode()).decode().rstrip("=") + return f"eyJhbGciOiJIUzI1NiJ9.{payload}.sig" + + +class _Resp: + status_code = 200 + + def __init__(self, data=None): + self._data = data or {} + self.text = "" + + def json(self): + return self._data + + +# --------------------------------------------------------------------- OCR + + +def _distance_calculator(monkeypatch): + """captcha_solver imports playwright at module level; stub the page type. + + The class under test never touches a page, so a placeholder is enough and + the real module still loads on a machine without the browser stack. + """ + if "playwright" not in sys.modules: + fake = types.ModuleType("playwright.sync_api") + fake.Page = object + pkg = types.ModuleType("playwright") + pkg.sync_api = fake + monkeypatch.setitem(sys.modules, "playwright", pkg) + monkeypatch.setitem(sys.modules, "playwright.sync_api", fake) + solver = _load_script("bugfix_solver", GEN / "captcha_solver.py") + return solver + + +def test_need_ocr_returns_none_when_the_lazy_factory_raises(monkeypatch, caplog): + solver = _distance_calculator(monkeypatch) + + def factory(): + raise RuntimeError("ddddocr is not installed (pip install ddddocr)") + + calc = solver.DistanceCalculator(ocr=factory, + logger=logging.getLogger("bugfix-ocr")) + with caplog.at_level(logging.WARNING): + assert calc._need_ocr() is None + assert "ddddocr is not installed" in caplog.text + + +def test_need_ocr_memoizes_the_lazy_factory_result(monkeypatch): + solver = _distance_calculator(monkeypatch) + calls = [] + ocr = SimpleNamespace(slide_match=lambda *a, **k: None) + + def factory(): + calls.append(1) + return ocr + + calc = solver.DistanceCalculator(ocr=factory, + logger=logging.getLogger("bugfix-ocr")) + assert calc._need_ocr() is ocr + assert calc._need_ocr() is ocr + assert calls == [1] + + +# ------------------------------------------------- per-account device / timezone + + +def _invite_claim(): + return _module("invite_claim") + + +def test_login_telemetry_sends_the_per_account_device_id(monkeypatch): + ic = _invite_claim() + if ic is None: + pytest.skip("invite_claim unavailable") + sent = [] + + def fake_request(method, url, **kw): + sent.append(kw) + return _Resp({"code": 0}) + + monkeypatch.setattr(ic, "_oauth_request", fake_request) + monkeypatch.setattr(ic, "client_timezone", lambda: "UTC") + token = _jwt() + assert ic.report_login_device(token, log=lambda m: None) is True + + mid = ic.per_account_device_mid(token) + assert mid != ic.APP_HEADERS["x-device-mid"] # not the shared capture + assert len(sent) == 2 # app_login_ck + success + for kw in sent: + assert kw["headers"]["x-device-mid"] == mid + assert kw["json"]["device_mid"] == mid + + +def test_device_login_sends_the_per_account_device_id(monkeypatch): + ic = _invite_claim() + if ic is None: + pytest.skip("invite_claim unavailable") + sent = [] + + def fake_request(method, url, **kw): + sent.append((url, kw)) + if "cli/init" in url: + return _Resp({"data": {"flow_id": "f1", "poll_token": "p1", + "authorize_url": "https://x/?state=S1"}}) + if "oauth/authorize" in url: + return _Resp({"redirect_url": "https://x/?code=code-abc123"}) + if "cli/poll" in url: + return _Resp({"data": {"status": "ready", "token": "ZJWT", + "user": {"user_id": "u1"}, "zai": {}}}) + return _Resp() + + monkeypatch.setattr(ic, "_oauth_request", fake_request) + monkeypatch.setattr(ic, "report_login_device", lambda *a, **k: True) + monkeypatch.setattr(ic, "client_timezone", lambda: "UTC") + assert ic.device_login("chat-token", log=lambda m: None) == ("ZJWT", False) + + mid = ic.per_account_device_mid("chat-token") + assert mid != ic.APP_HEADERS["x-device-mid"] + flows = {marker: kw for url, kw in sent + for marker in ("cli/init", "cli/poll", "callback") if marker in url} + assert set(flows) == {"cli/init", "cli/poll", "callback"} + for kw in flows.values(): + assert kw["headers"]["x-device-mid"] == mid + + +def test_login_telemetry_header_and_body_share_the_client_timezone(monkeypatch): + ic = _invite_claim() + if ic is None: + pytest.skip("invite_claim unavailable") + sent = [] + + def fake_request(method, url, **kw): + sent.append(kw) + return _Resp({"code": 0}) + + monkeypatch.setattr(ic, "_oauth_request", fake_request) + monkeypatch.setattr(ic, "client_timezone", lambda: "Pacific/Auckland") + assert ic.report_login_device(_jwt(), log=lambda m: None) is True + + assert len(sent) == 2 + for kw in sent: + assert kw["headers"]["x-client-timezone"] == "Pacific/Auckland" + assert kw["json"]["client_timezone"] == "Pacific/Auckland" + + +def test_plan_events_use_the_client_timezone(monkeypatch): + pc = _module("plan_check") + monkeypatch.delenv("ZAI_ACTIVITY_EVENTS", raising=False) + monkeypatch.setattr(pc, "client_timezone", lambda: "America/Chicago") + + events = pc._plan_events(_jwt()) + + assert events + assert {event["client_timezone"] for event in events} == {"America/Chicago"} + + +def test_claim500_headers_use_the_client_timezone(monkeypatch): + c500 = _module("claim500") + if c500 is None: + pytest.skip("claim500 unavailable") + monkeypatch.setattr(c500, "client_timezone", lambda: "America/New_York") + + assert c500._headers("zjwt")["x-client-timezone"] == "America/New_York" + + +# ------------------------------------------------------------------ batches + + +def _export_module(monkeypatch, tmp_path, accounts, get_jwt): + stub = SimpleNamespace( + get_zcode_jwt=get_jwt, + get_plan=lambda zjwt, **kw: {"plans": ["zcode-v3-start-plan-1024"], + "starter": True}, + is_start_plan=lambda plan_id: True, + DEVICE_MID="58aca3b3-3239-4cb7-9c70-df3e5488fb99", + per_account_device_mid=lambda token: "mid-" + token[-6:], + ) + monkeypatch.setitem(sys.modules, "plan_check", stub) + monkeypatch.setenv("ZAI_OUTPUT_DIR", str(tmp_path / "out")) + monkeypatch.setenv("ZCODE_GATEWAY_ACCOUNTS", str(tmp_path / "accounts")) + monkeypatch.setattr(sys, "argv", ["export_starter.py"]) + results = tmp_path / "out" / "results" + results.mkdir(parents=True) + (results / "accounts.txt").write_text("\n".join(accounts) + "\n", + encoding="utf-8") + return _load_script("bugfix_export", GEN / "export_starter.py") + + +def test_export_snapshot_uses_the_per_account_device_id(tmp_path, monkeypatch): + token = _jwt("u9") + module = _export_module(monkeypatch, tmp_path, + ["one@example.test:pw:" + token], + lambda chat_token, log=None: token) + + module.main() + + snap = (tmp_path / "accounts" / "one@example.test.txt").read_text( + encoding="utf-8") + fields = dict(line.split(":", 1) for line in snap.splitlines() + if line and not line.startswith("#") and ":" in line) + assert fields["device_mid"].strip() == "mid-" + token[-6:] + assert fields["device_mid"].strip() != "58aca3b3-3239-4cb7-9c70-df3e5488fb99" + + +def test_export_continues_after_one_account_fails(tmp_path, monkeypatch, + capsys): + accounts = ["first@example.test:pw:eyJfirst", + "second@example.test:pw:eyJsecond"] + + def get_jwt(chat_token, log=None): + if chat_token == "eyJfirst": + raise RuntimeError("oauth poll never became ready") + return _jwt("u2") + + module = _export_module(monkeypatch, tmp_path, accounts, get_jwt) + + module.main() # must not raise out of the batch + + written = sorted(p.name for p in (tmp_path / "accounts").iterdir()) + assert written == ["second@example.test.txt"] + out = capsys.readouterr().out + assert "RuntimeError" in out and "failed" in out + + +def test_recheck_continues_after_one_account_fails(tmp_path, monkeypatch): + rp = _module("recheck_plans") + if rp is None: + pytest.skip("recheck_plans unavailable") + results = tmp_path / "results" + results.mkdir() + (results / "accounts.txt").write_text( + "bad@example.test:pw:eyJbad\nok@example.test:pw:eyJok\n", + encoding="utf-8") + monkeypatch.setenv("ZAI_OUTPUT_DIR", str(tmp_path)) + monkeypatch.setattr(sys, "argv", ["recheck_plans.py"]) + calls = [] + + def get_jwt(token, log=None): + calls.append(token) + if token == "eyJbad": + raise RuntimeError("poll never became ready") + return "ZJWT" + + monkeypatch.setattr(rp, "get_zcode_jwt", get_jwt) + monkeypatch.setattr(rp, "get_plan", + lambda zjwt, log=None: {"plans": ["zcode-v3-start-plan-1024"]}) + monkeypatch.setattr(rp, "is_start_plan", lambda plan_id: True) + + assert rp.main() == 0 + assert calls == ["eyJbad", "eyJok"] + + rows = [json.loads(line) for line in + (results / "plans_recheck.jsonl").read_text( + encoding="utf-8").splitlines()] + assert [row["email"] for row in rows] == ["bad@example.test", + "ok@example.test"] + assert "RuntimeError" in rows[0]["error"] + assert rows[1]["starter"] is True + + +# ------------------------------------------------------------------- proxies + + +def test_portless_proxy_is_not_handed_to_playwright(): + ic = _invite_claim() + if ic is None: + pytest.skip("invite_claim unavailable") + + assert ic._pw_proxy("socks5://1.2.3.4") is None # was "...:None" + assert ic._pw_proxy("http://1.2.3.4") is None + assert ic._pw_proxy("0") is None + assert ic._pw_proxy("http://user:pw@1.2.3.4:8080") == { + "server": "http://1.2.3.4:8080", "username": "user", "password": "pw"} + + +def test_endpoint_rejects_leading_zero_octets(monkeypatch): + pp = _module("proxy_parse") + monkeypatch.delenv("ZAI_ALLOW_LOOPBACK", raising=False) + monkeypatch.delenv("PROXYHIVE_ALLOW_LOOPBACK", raising=False) + monkeypatch.delenv("ZAI_ALLOW_PRIVATE", raising=False) + + # inet_aton reads a leading zero as octal: 0177.0.0.1 IS 127.0.0.1 + assert pp.valid_endpoint("0177.0.0.1", 8080) is False + assert pp.valid_endpoint("0127.0.0.1", 8080) is False + assert pp.parse_proxy_list("0177.0.0.1:8080") == [] + assert pp.valid_endpoint("127.0.0.1", 8080) is False + assert pp.valid_endpoint("93.184.216.34", 8080) is True + assert pp.valid_endpoint("8.8.8.8", 1080) is True diff --git a/tests/test_gui_bugfixes.py b/tests/test_gui_bugfixes.py new file mode 100644 index 0000000..7881662 --- /dev/null +++ b/tests/test_gui_bugfixes.py @@ -0,0 +1,400 @@ +"""Regressions for the desktop client lifecycle, atomic config writes and the +live smoke script's teardown order.""" +from __future__ import annotations + +import importlib.util +import json +import os +import threading +import urllib.request +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + +from zcode_gateway import gui + +ROOT = Path(__file__).resolve().parents[1] + + +# --------------------------------------------------------------------------- # +# lifecycle: a user Stop wins over a pending restart +# --------------------------------------------------------------------------- # + + +class StubServer: + def __init__(self) -> None: + self.host = "127.0.0.1" + self.port = 8788 + self.error = None + self.accounts_loaded = 0 + self.effective_accounts_dir = None + self.started: list[Path | None] = [] + + @property + def running(self) -> bool: + return False + + @property + def base_url(self) -> str: + return f"http://{self.host}:{self.port}" + + def start(self, cfg_path: Path | None) -> None: + self.started.append(cfg_path) + + def stop(self, wait: float = 5.0) -> None: + pass + + def wait_stopped(self, timeout: float = 10.0) -> bool: + return True + + def wait_ready(self, timeout: float = 25.0) -> bool: + return False + + +def flush_ui(pending): + """Run the callbacks a worker thread posted, as the Tk main loop would.""" + while pending: + func, args = pending.pop(0) + func(*args) + + +@pytest.fixture(scope="module") +def tk_app(tmp_path_factory): + # One Tk interpreter per process: destroying the last root and creating a + # new one intermittently fails with "Can't find a usable tk.tcl". + tkinter = pytest.importorskip("tkinter") + pending: list[tuple] = [] + real_after = tkinter.Misc.after + + def after(self, ms, func=None, *args): + # tkinter cannot marshal a call from a worker thread unless the real + # main loop is running; queue it so the test thread runs it instead + if threading.current_thread() is threading.main_thread(): + return real_after(self, ms, func, *args) + assert func is not None + pending.append((func, args)) + return "" + + cfg = tmp_path_factory.mktemp("gui") / "config.yaml" + cfg.write_text(gui._default_config_yaml(), encoding="utf-8") + with pytest.MonkeyPatch.context() as patch: + patch.setattr(gui, "find_config", lambda: cfg) + patch.setattr(gui, "messagebox", SimpleNamespace( + showerror=lambda *a, **k: None, askyesno=lambda *a, **k: True)) + # the lifecycle under test never talks to a real gateway: keep the + # background pollers from reaching this machine's own processes + patch.setattr(gui.App, "poll", lambda self, force=False: None) + patch.setattr(gui.App, "_drain_queue", lambda self: None) + patch.setattr(gui.App, "_refresh_zcode_status", lambda self: None) + patch.setattr(gui.App, "_check_client_reasoning", lambda self: None) + patch.setattr(gui.App, "after", after) + try: + app = gui.App(autostart=False) + except tkinter.TclError as exc: + pytest.skip(f"no Tk display available: {exc}") + app.withdraw() + try: + yield app, pending + finally: + app.destroy() + + +@pytest.fixture +def lifecycle_app(tk_app): + app, pending = tk_app + app._lifecycle = 0 + app._stopping = False + app._restart_pending = False + app._restart_message = None + app._last_state = None + app._last_avail = None + app.server = StubServer() + pending.clear() + return app, pending + + +@pytest.fixture +def recording_threads(monkeypatch): + threads: list[threading.Thread] = [] + real_thread = threading.Thread + + class Recording(real_thread): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + threads.append(self) + + monkeypatch.setattr(gui.threading, "Thread", Recording) + return threads + + +def test_user_stop_cancels_pending_restart(lifecycle_app, recording_threads): + app, pending = lifecycle_app + release = threading.Event() + app.server.wait_stopped = lambda timeout=10.0: release.wait(5.0) + + app.restart_server("Saved. Gateway restarted.") + restart_thread, = recording_threads + app.stop_everything() # the user hits Stop while the restart waits + release.set() + restart_thread.join(5) + assert not restart_thread.is_alive() + + flush_ui(pending) # deliver whatever the restart thread queued + assert app.server.started == [] + + +def test_restart_still_restarts_after_a_clean_stop(lifecycle_app, recording_threads): + app, pending = lifecycle_app + + app.restart_server("Saved. Gateway restarted.") + restart_thread, = recording_threads + restart_thread.join(5) + assert not restart_thread.is_alive() + + flush_ui(pending) + assert app.server.started == [app.cfg_path] + assert app._stopping is False + + +# --------------------------------------------------------------------------- # +# atomic config writes +# --------------------------------------------------------------------------- # + + +def test_config_write_survives_a_midwrite_failure(tmp_path, monkeypatch): + path = tmp_path / "config.yaml" + original = "server:\n admin_key: keep-me\n client_keys:\n - key: keep-me-too\n" + path.write_text(original, encoding="utf-8") + real_fdopen = os.fdopen + + def exploding_fdopen(fd, *args, **kwargs): + stream = real_fdopen(fd, *args, **kwargs) + + class Failing: + def write(self, text): + stream.write(text[: len(text) // 2]) + stream.flush() + raise OSError("disk full") + + def flush(self): + stream.flush() + + def fileno(self): + return stream.fileno() + + def __enter__(self): + return self + + def __exit__(self, *exc): + stream.close() + return False + + return Failing() + + monkeypatch.setattr("os.fdopen", exploding_fdopen) + + with pytest.raises(OSError): + gui._write_config_atomic(path, "server:\n admin_key: new\n") + + assert path.read_text(encoding="utf-8") == original + assert [p.name for p in tmp_path.iterdir()] == ["config.yaml"] + + +def test_config_write_replaces_the_file_atomically(tmp_path, monkeypatch): + path = tmp_path / "config.yaml" + path.write_text("old", encoding="utf-8") + real_replace = os.replace + replaced = [] + + def tracking_replace(src, dst): + replaced.append((Path(src), Path(dst))) + return real_replace(src, dst) + + monkeypatch.setattr("os.replace", tracking_replace) + + gui._write_config_atomic(path, "new") + + assert [(src.parent, dst) for src, dst in replaced] == [(tmp_path, path)] + assert replaced[0][0] != path + assert path.read_text(encoding="utf-8") == "new" + assert [p.name for p in tmp_path.iterdir()] == ["config.yaml"] + + +# --------------------------------------------------------------------------- # +# server controller: stop during startup, readiness against the right URL +# --------------------------------------------------------------------------- # + + +def _write_config(tmp_path: Path, port: int) -> Path: + accounts = tmp_path / "accounts" + accounts.mkdir() + cfg = tmp_path / "config.yaml" + cfg.write_text(yaml.safe_dump({ + "server": {"host": "127.0.0.1", "port": port}, + "accounts": {"dir": str(accounts)}, + }), encoding="utf-8") + return cfg + + +def test_stop_during_startup_prevents_binding(tmp_path, monkeypatch): + import uvicorn + + from zcode_gateway import app as app_module + + entered = threading.Event() + release = threading.Event() + + def slow_create_app(cfg): + entered.set() + assert release.wait(5) + return object() + + monkeypatch.setattr(app_module, "create_app", slow_create_app) + created = [] + + class FakeServer: + def __init__(self, config): + created.append(config) + + def run(self): + pass + + monkeypatch.setattr(uvicorn, "Server", FakeServer) + + controller = gui.ServerController() + controller.start(_write_config(tmp_path, 9124)) + assert entered.wait(5) + + controller.stop(wait=0.1) # uvicorn is still being imported/built + release.set() + + assert controller.wait_stopped(5) is True + assert created == [] + assert controller.error is None + + +class Healthy: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def test_wait_ready_needs_the_config_resolved(monkeypatch): + controller = gui.ServerController() + monkeypatch.setattr(urllib.request, "urlopen", + lambda *a, **k: Healthy()) + + # something answers /health on the default URL, but this controller never + # resolved its own host/port, so the listener is not ours + assert controller.wait_ready(timeout=0.2) is False + + +def test_wait_ready_probes_the_configured_port(tmp_path, monkeypatch): + import uvicorn + + from zcode_gateway import app as app_module + + monkeypatch.setattr(uvicorn, "Config", lambda *a, **k: object()) + + class FakeServer: + def __init__(self, config): + pass + + def run(self): + pass + + monkeypatch.setattr(uvicorn, "Server", FakeServer) + monkeypatch.setattr(app_module, "create_app", lambda cfg: object()) + probed = [] + monkeypatch.setattr(urllib.request, "urlopen", + lambda url, timeout=None: (probed.append(url), Healthy())[1]) + + controller = gui.ServerController() + controller.start(_write_config(tmp_path, 9123)) + try: + assert controller.wait_ready(timeout=5) is True + assert probed[0] == "http://127.0.0.1:9123/health" + finally: + controller.stop() + + +# --------------------------------------------------------------------------- # +# live smoke script: drain the service before closing the store +# --------------------------------------------------------------------------- # + + +def _live_module(): + path = ROOT / "scripts" / "smoke" / "live.py" + spec = importlib.util.spec_from_file_location("live_smoke", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_live_smoke_drains_the_service_before_closing_the_store(monkeypatch): + live = _live_module() + events = [] + + class FakePool: + def __init__(self, cfg, accounts): + pass + + class FakeAccount: + def __init__(self): + self.id = "account-1" + self.notes = {"quotas": {"GLM-5.3-Flash": {"remaining": 1, "total": 2}}} + + class FakeTokens: + def __init__(self, cfg, pool): + pass + + async def check_account(self, account): + return {"ok": True} + + async def aclose(self): + pass + + class FakeClient: + def __init__(self, cfg): + pass + + async def aclose(self): + pass + + class FakeService: + def __init__(self, *args): + pass + + async def complete(self, payload, **kwargs): + return SimpleNamespace(raw=json.dumps( + {"stop_reason": "end_turn", "usage": {"input_tokens": 1}})) + + async def aclose(self): + events.append("service.aclose") + + class FakeStore: + def __init__(self, path): + pass + + def close(self): + events.append("store.close") + + monkeypatch.setattr(live, "load_config", lambda path: SimpleNamespace( + accounts=None, pool=None, upstream=None)) + monkeypatch.setattr(live, "load_accounts", lambda cfg: [FakeAccount()]) + monkeypatch.setattr(live, "PoolManager", FakePool) + monkeypatch.setattr(live, "TokenManager", FakeTokens) + monkeypatch.setattr(live, "UpstreamClient", FakeClient) + monkeypatch.setattr(live, "ChatService", FakeService) + monkeypatch.setattr(live, "Store", FakeStore) + monkeypatch.setattr(live, "JsonlCapture", lambda *a: None) + monkeypatch.setattr(live, "Registry", lambda: None) + + status = live.asyncio.run(live.run(SimpleNamespace(config="config.yaml", infer=True))) + + assert status == 0 + assert events == ["service.aclose", "store.close"] diff --git a/tests/test_pool_bugfixes.py b/tests/test_pool_bugfixes.py new file mode 100644 index 0000000..f744974 --- /dev/null +++ b/tests/test_pool_bugfixes.py @@ -0,0 +1,204 @@ +"""Regressions for retirement stickiness, strike-budget resets, the edge-block +callback, captcha ticket reuse and notification dedupe ordering.""" +from __future__ import annotations + +import time + +import pytest + +from zcode_gateway import notify +from zcode_gateway.breaker import CLEAN_AFTER_S, EdgeBlockBreaker +from zcode_gateway.captcha_tickets import CaptchaTicket, CaptchaTicketPool +from zcode_gateway.config import Config +from zcode_gateway.pool import Account, AccountState, PoolManager, Router +from zcode_gateway.tokens import TokenManager + +MODEL = "GLM-5.3" +EDGE_403 = ("403 Forbidden" + "

Your client does not have permission to get URL " + "/api/v1/chat/completions from this server.

" + "") + + +class _Clock: + def __init__(self, now: float) -> None: + self.now = now + + def time(self) -> float: + return self.now + + +def _pool(*accounts: Account) -> tuple[PoolManager, Router]: + cfg = Config().pool + return PoolManager(cfg, list(accounts)), Router(cfg) + + +# --------------------------------------------------------------------------- # +# EXHAUSTED is quota retirement, not a transient error +# --------------------------------------------------------------------------- # + +@pytest.mark.asyncio +async def test_cool_keeps_exhausted_account_retired(): + account = Account(id="drained", access_token="token", + state=AccountState.EXHAUSTED) + pool, router = _pool(account) + await pool.cool(account, reason="mid-flight upstream_error") + assert account.state is AccountState.EXHAUSTED + assert not account.is_ready() + assert router.candidates([account], set()) == [] + await pool.aclose() + + +@pytest.mark.asyncio +async def test_kill_after_strikes_keeps_exhausted_account_retired(): + account = Account(id="drained", access_token="token", + state=AccountState.EXHAUSTED) + pool, router = _pool(account) + await pool.kill_after_strikes(account, "token revoked") + assert account.state is AccountState.EXHAUSTED + assert not account.is_ready() + assert router.candidates([account], set()) == [] + await pool.aclose() + + +# --------------------------------------------------------------------------- # +# credential replacement / admin enable give a fresh strike budget +# --------------------------------------------------------------------------- # + +@pytest.mark.asyncio +async def test_upsert_resets_banked_quota_strikes_and_keeps_model_caps(): + existing = Account(id="dummy", access_token="old", quota_strikes=3, + auth_strikes=2) + existing.model_caps[MODEL] = time.time() + 600 + pool, _ = _pool(existing) + await pool.upsert(Account(id="dummy", access_token="new")) + assert existing.access_token == "new" + assert existing.quota_strikes == 0 + assert existing.auth_strikes == 0 + assert existing.model_caps[MODEL] > time.time() + await pool.aclose() + + +@pytest.mark.asyncio +async def test_enable_resets_banked_quota_strikes(): + account = Account(id="dummy", access_token="token", + state=AccountState.EXHAUSTED, quota_strikes=3, + auth_strikes=3) + pool, _ = _pool(account) + assert await pool.enable("dummy") + assert account.state is AccountState.READY + assert account.quota_strikes == 0 + assert account.auth_strikes == 0 + await pool.aclose() + + +@pytest.mark.asyncio +async def test_apply_credentials_reset_clears_banked_quota_strikes(): + account = Account(id="dummy", access_token="old", quota_strikes=3, + state=AccountState.EXHAUSTED) + pool, _ = _pool(account) + manager = TokenManager(Config(), pool) + await manager.apply_credentials(account, access_token="new") + assert account.quota_strikes == 0 + assert account.state is AccountState.READY + await manager.aclose() + await pool.aclose() + + +# --------------------------------------------------------------------------- # +# upsert keeps live state +# --------------------------------------------------------------------------- # + +@pytest.mark.asyncio +async def test_upsert_preserves_cooling_state_and_cooldown(): + existing = Account(id="dummy", access_token="old", + state=AccountState.COOLING) + existing.cooldown_until = time.time() + 300 + cooling_until = existing.cooldown_until + pool, _ = _pool(existing) + await pool.upsert(Account(id="dummy", access_token="new")) + assert existing.access_token == "new" + assert existing.state is AccountState.COOLING + assert existing.cooldown_until == cooling_until + await pool.aclose() + + +@pytest.mark.asyncio +async def test_upsert_preserves_exhausted_state(): + existing = Account(id="dummy", access_token="old", + state=AccountState.EXHAUSTED) + pool, _ = _pool(existing) + await pool.upsert(Account(id="dummy", access_token="new")) + assert existing.state is AccountState.EXHAUSTED + await pool.aclose() + + +# --------------------------------------------------------------------------- # +# edge-block callback fires once per episode +# --------------------------------------------------------------------------- # + +def test_breaker_callback_fires_once_per_episode(monkeypatch): + clock = _Clock(1000.0) + monkeypatch.setattr("zcode_gateway.breaker.time", clock) + breaker = EdgeBlockBreaker() + fired = [] + breaker.on_block = fired.append + + assert breaker.note_edge_block(EDGE_403) is True + assert breaker.note_edge_block(EDGE_403) is True + assert breaker.note_edge_block(EDGE_403) is True + assert fired == [1] + assert breaker.consecutive == 3 + + clock.now += CLEAN_AFTER_S + 1 # stale episode: the next block is new + assert breaker.note_edge_block(EDGE_403) is True + assert fired == [1, 1] + assert breaker.consecutive == 1 + + +# --------------------------------------------------------------------------- # +# captcha take() keeps the tickets it did not hand out +# --------------------------------------------------------------------------- # + +def test_take_keeps_remaining_fresh_tickets_and_drops_stale(): + clock = _Clock(100.0) + pool = CaptchaTicketPool(size=3, ttl_seconds=75, mint=lambda: None, + clock=clock.time) + pool.put(CaptchaTicket("stale-param-value-xxxxxxxx", "sgp", minted_at=10.0)) + pool.put(CaptchaTicket("first-param-value-xxxxxxxx", "sgp", minted_at=100.0)) + pool.put(CaptchaTicket("second-param-value-xxxxxxxx", "sgp", minted_at=100.0)) + + ticket = pool.take() + assert ticket is not None and ticket.param.startswith("second") + assert pool.stats()["ready"] == 1 # stale one discarded, sibling kept + + ticket = pool.take() + assert ticket is not None and ticket.param.startswith("first") + assert pool.stats()["ready"] == 0 + assert pool.take() is None + + +# --------------------------------------------------------------------------- # +# notification dedupe does not depend on deque position +# --------------------------------------------------------------------------- # + +def test_notify_dedupe_bumps_entry_behind_a_stale_one(monkeypatch): + clock = _Clock(1000.0) + monkeypatch.setattr("zcode_gateway.notify.time", clock) + center = notify.NotificationCenter() + + first = center.emit("dedupe_case", "A") + clock.now += 1 + center.emit("dedupe_case", "B") + clock.now += 54 # 55 s after the first A + assert center.emit("dedupe_case", "A") == first + clock.now += 5 # B is now 60 s old, A was bumped + center.emit("dedupe_case", "C") + clock.now += 2 + + assert center.emit("dedupe_case", "A") == first + events = center.list() + assert [e["message"] for e in events] == ["A", "B", "C"] + assert events[0]["id"] == first + assert events[0]["count"] == 3 + assert events[0]["ts"] == 1062 diff --git a/tests/test_relays.py b/tests/test_relays.py index 0dc13c6..1a5a736 100644 --- a/tests/test_relays.py +++ b/tests/test_relays.py @@ -3,8 +3,10 @@ import json from pathlib import Path +from types import SimpleNamespace from unittest.mock import MagicMock +import httpx import pytest from zcode_gateway.relay_deploy import ( @@ -15,7 +17,7 @@ ping_relay, ) from zcode_gateway.relay_manager import RelayManager -from zcode_gateway.upstream import UpstreamClient, UpstreamConfig +from zcode_gateway.upstream import RelayTransportError, UpstreamClient, UpstreamConfig def test_relay_manager_defaults(tmp_path: Path): @@ -249,3 +251,113 @@ def test_all_relays_failed_still_routes(tmp_path: Path): mgr.mark_relay_failed("https://cf2.workers.dev") assert mgr.get_relay_for_model("GLM-5.3") in { "https://cf1.workers.dev", "https://cf2.workers.dev"} + + +# --------------------------------------------------------------------------- # +# retry rotation inside one UpstreamClient.send call +# --------------------------------------------------------------------------- # + +def _upstream_client(mgr: RelayManager): + client = UpstreamClient.__new__(UpstreamClient) # no real httpx pool + client.cfg = UpstreamConfig(base_url="https://zcode.z.ai/api/v1", + chat_path="/zcode-plan/anthropic/v1/messages") + client.pacing, client.relay_manager = None, mgr + return client + + +def _record_failures(client): + """Replace the transport: record each target URL, fail every send.""" + calls: list[str] = [] + + def build_request(method, url, **kwargs): + calls.append(url) + return object() + + async def refuse(request, stream): + raise httpx.ConnectError("connection refused") + + client._client = SimpleNamespace(build_request=build_request, send=refuse) + return calls + + +@pytest.mark.asyncio +async def test_a_failed_relay_is_never_retried_within_one_call(tmp_path: Path): + """cf2 is cooling down, so every pick lands on the active relay cf1. Once + cf1 has failed, the retry must not hammer it a second time.""" + mgr = RelayManager(data_dir=tmp_path) + mgr.add_relay("cf1", "https://cf1.workers.dev", "cloudflare") + mgr.add_relay("cf2", "https://cf2.workers.dev", "cloudflare") + mgr.update_options(enabled=True, strategy="single") + mgr.mark_relay_failed("https://cf2.workers.dev") + client = _upstream_client(mgr) + calls = _record_failures(client) + + with pytest.raises(RelayTransportError): + await client.send({"authorization": "Bearer t"}, {"model": "GLM-5.3"}, True) + assert calls == ["https://cf1.workers.dev"] + + +@pytest.mark.asyncio +async def test_relay_rotation_still_tries_the_next_relay(tmp_path: Path): + """A relay that has NOT been tried is still rotated to.""" + mgr = RelayManager(data_dir=tmp_path) + mgr.add_relay("cf1", "https://cf1.workers.dev", "cloudflare") + mgr.add_relay("cf2", "https://cf2.workers.dev", "cloudflare") + mgr.update_options(enabled=True, strategy="round_robin") + client = _upstream_client(mgr) + calls = _record_failures(client) + + with pytest.raises(RelayTransportError): + await client.send({"authorization": "Bearer t"}, {"model": "GLM-5.3"}, True) + assert calls == ["https://cf1.workers.dev", "https://cf2.workers.dev"] + + +# --------------------------------------------------------------------------- # +# relays.json must survive a read failure +# --------------------------------------------------------------------------- # + +def test_unreadable_relays_file_is_not_saved_over_as_defaults(tmp_path: Path, monkeypatch): + """A transient read error used to cache DEFAULT settings, so the next save + wrote defaults over a perfectly good relays.json.""" + mgr = RelayManager(data_dir=tmp_path) + entry = mgr.add_relay("cf", "https://cf.workers.dev", "cloudflare") + fresh = RelayManager(data_dir=tmp_path) # nothing cached yet + real_read = Path.read_text + reads = [] + + def flaky_read(self, *args, **kwargs): + reads.append(self.name) + if len(reads) == 1: + raise PermissionError("relays.json is locked") + return real_read(self, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", flaky_read) + assert fresh.get_settings()["relays"] == [] # defaults for this read only + fresh.update_options(enabled=True) + on_disk = json.loads(real_read(tmp_path / "relays.json", encoding="utf-8")) + assert [r["id"] for r in on_disk["relays"]] == [entry["id"]] + assert fresh.get_settings()["enabled"] is True + + +def test_corrupt_relays_file_is_reread_once_repaired(tmp_path: Path): + """Garbage must not be cached as authoritative defaults: a repaired file + has to be picked up without restarting the gateway.""" + (tmp_path / "relays.json").write_text("{ truncated", encoding="utf-8") + mgr = RelayManager(data_dir=tmp_path) + assert mgr.get_settings()["relays"] == [] + (tmp_path / "relays.json").write_text(json.dumps({ + "enabled": True, "models": ["GLM-5.3"], + "relays": [{"id": "relay-1", "url": "https://cf.workers.dev"}]}), encoding="utf-8") + assert mgr.get_settings()["relays"][0]["id"] == "relay-1" + + +# --------------------------------------------------------------------------- # +# option shapes from the dashboard +# --------------------------------------------------------------------------- # + +def test_update_options_accepts_a_single_model_string(tmp_path: Path): + mgr = RelayManager(data_dir=tmp_path) + mgr.add_relay("cf", "https://cf.workers.dev", "cloudflare") + mgr.update_options(models="GLM-5.3", enabled=True) + assert mgr.get_settings()["models"] == ["GLM-5.3"] + assert mgr.get_relay_for_model("GLM-5.3") == "https://cf.workers.dev" diff --git a/tests/test_sse_framing.py b/tests/test_sse_framing.py new file mode 100644 index 0000000..8c36074 --- /dev/null +++ b/tests/test_sse_framing.py @@ -0,0 +1,49 @@ +"""SSE framing must split on real line terminators only. + +A JSON payload serialized with ensure_ascii=False can carry U+2028 (LINE +SEPARATOR), U+2029 (PARAGRAPH SEPARATOR) and U+0085 (NEL) raw inside string +values. str.splitlines() treats those as line breaks, which cut the payload +mid-string: a 500 on non-streaming /v1/chat/completions, a mid-stream 502 +invalid_stream, and a stream with real content reported as empty. +""" +from __future__ import annotations + +import json + +import pytest + +from zcode_gateway.translate_anthropic import extract_stream_error, stream_had_content +from zcode_gateway.upstream import iter_sse_events + +UNICODE_LINE_BREAKS = ["\u2028", "\u2029", "\u0085"] + + +def _sse(*events, end="\r\n\r\n"): + return "".join("data: " + json.dumps(e, ensure_ascii=False) + end for e in events) + + +@pytest.mark.parametrize("sep", UNICODE_LINE_BREAKS) +def test_line_separator_inside_payload_survives_parsing(sep): + text = f"before{sep}after" + frames = _sse({"id": "chatcmpl-1", "choices": [{"delta": {"content": text}}]}, + {"id": "chatcmpl-1", "choices": [{"delta": {"content": "next"}}]}) + events = iter_sse_events(frames) + assert [e["choices"][0]["delta"]["content"] for e in events] == [text, "next"] + + +@pytest.mark.parametrize("sep", UNICODE_LINE_BREAKS) +def test_error_frame_message_with_line_separator_is_detected(sep): + message = f"stream died{sep}retry once" + error = extract_stream_error(_sse( + {"error": {"code": "stream_initialization_failed", "message": message}}, end="\n\n")) + assert error == {"code": "stream_initialization_failed", "message": message, + "request_id": None, "type": "stream_error"} + + +@pytest.mark.parametrize("sep", UNICODE_LINE_BREAKS) +def test_content_with_line_separator_counts_as_content(sep): + assert stream_had_content(_sse( + {"choices": [{"delta": {"content": f"a{sep}b"}}]}, end="\n\n")) + # role-only chunks still carry no content + assert not stream_had_content(_sse( + {"choices": [{"delta": {"role": "assistant"}}]}, end="\n\n")) diff --git a/zcode_gateway/api_admin.py b/zcode_gateway/api_admin.py index 53815b0..f491732 100644 --- a/zcode_gateway/api_admin.py +++ b/zcode_gateway/api_admin.py @@ -121,6 +121,9 @@ async def add_account( source="admin", ) await state.pool.upsert(account) + # re-adding an account is an explicit revive with fresh credentials; upsert + # keeps a parked state, so revival is requested here instead of implied + await state.pool.enable(account.id) return {"added": account.to_public()} @@ -255,6 +258,8 @@ async def import_jwt_account(request: Request, body: dict[str, Any] = Body(...), except (OSError, ValueError): raise HTTPException(500, "Could not save account snapshot; nothing added to pool") from None await state.pool.upsert(account) + # an explicit import of fresh credentials revives a parked account + await state.pool.enable(ident) current = await state.pool.find(ident) current.balance_checked_at = account.balance_checked_at current.plan_checked_at = account.plan_checked_at @@ -615,7 +620,7 @@ async def claim_stop(request: Request, body: dict = Body(...), mgr = _generator(request) if mgr is None: raise HTTPException(status_code=404, detail="generator manager unavailable") - ok = mgr.stop_claim(name) + ok = await asyncio.to_thread(mgr.stop_claim, name) if not ok: raise HTTPException(status_code=404, detail="no such claim") return {"stopped": name} @@ -629,7 +634,7 @@ async def claim_remove(request: Request, body: dict = Body(...), mgr = _generator(request) if mgr is None: raise HTTPException(status_code=404, detail="generator manager unavailable") - ok = mgr.remove_claim(name) + ok = await asyncio.to_thread(mgr.remove_claim, name) if not ok: raise HTTPException(status_code=404, detail="no such claim") return {"removed": name} @@ -711,6 +716,15 @@ async def proxy_auto_verify(request: Request, body: dict = Body(...), mgr = _generator(request) enabled = bool(body.get("enabled")) interval = body.get("interval_min") + if interval not in (None, ""): + try: + interval = int(interval) + except (TypeError, ValueError): + raise HTTPException( + status_code=400, + detail="interval_min must be a whole number of minutes") from None + else: + interval = None return mgr.checker.set_auto_verify(enabled, interval) diff --git a/zcode_gateway/api_dash.py b/zcode_gateway/api_dash.py index e17344c..7a29afb 100644 --- a/zcode_gateway/api_dash.py +++ b/zcode_gateway/api_dash.py @@ -478,6 +478,9 @@ async def _add_and_probe(account): """New account: join the pool, then settle balance/paid-lane/plan now — a login that replaced an exhausted account must not read 'available'.""" await state.pool.upsert(account) + # logging in again is an explicit revive with fresh credentials; upsert + # keeps a parked state, so revival is requested here instead of implied + await state.pool.enable(account.id) try: canonical = await state.pool.find(account.id) if canonical is not None: @@ -703,7 +706,17 @@ async def _check_update_idle(state) -> None: raise HTTPException(409, "Stop Auto Generate before applying an update") if getattr(manager, "_claim_all_running", False): raise HTTPException(409, "Stop the claim batch before applying an update") - if any(not worker.done.is_set() for worker in manager.workers.values()) or any(job.alive() for job in manager.claims.values()) or manager.checker.alive(): + # workers/claims are mutated from generator threads: iterating them without + # the manager lock raised "dictionary changed size during iteration" + lock = getattr(manager, "_lock", None) + if lock is not None: + with lock: + workers = list(manager.workers.values()) + claims = list(manager.claims.values()) + else: + workers = list(manager.workers.values()) + claims = list(manager.claims.values()) + if any(not worker.done.is_set() for worker in workers) or any(job.alive() for job in claims) or manager.checker.alive(): raise HTTPException(409, "Generator, claim or proxy-check jobs are active; stop them before applying an update") @@ -962,10 +975,21 @@ async def generator_locate_set(request: Request, body: dict = Body(default={}), # rebuild the manager so the change takes effect immediately old = getattr(state, "generator", None) if old is not None: + def _teardown_old() -> None: + # stop_all() only stops workers/auto: the scheduler, the captcha + # mint thread and the proxy checker outlive it, so the replacement + # manager would run beside the old one (duplicate daily claims). + # shutdown() is the same teardown the process-exit paths apply. + scheduler = getattr(state, "claim_scheduler", None) + if scheduler is not None: + scheduler.stop() + old.shutdown() try: - old.stop_all() + await asyncio.to_thread(_teardown_old) except Exception: - pass + log.warning("generator: could not fully stop the previous manager", + exc_info=True) + state.claim_scheduler = None state.generator = None return {"ok": True, "path": str(root), "found": True, "applied": "live"} @@ -1608,12 +1632,12 @@ async def setup_status(request: Request, key: str = Depends(admin_key)) -> dict: async def generator_stop(request: Request, body: dict = Body(...), key: str = Depends(admin_key)) -> dict: name = body.get("name", "") - return {"stopped": _generator(request).stop_worker(name)} + return {"stopped": await asyncio.to_thread(_generator(request).stop_worker, name)} @router.post("/admin/dash/generator/stopall") async def generator_stopall(request: Request, key: str = Depends(admin_key)) -> dict: - _generator(request).stop_all() + await asyncio.to_thread(_generator(request).stop_all) return {"ok": True} @@ -1630,7 +1654,7 @@ async def generator_shutdown(request: Request, key: str = Depends(admin_key)) -> async def generator_worker_remove(request: Request, body: dict = Body(...), key: str = Depends(admin_key)) -> dict: name = str(body.get("name") or "") - ok = _generator(request).remove_worker(name) + ok = await asyncio.to_thread(_generator(request).remove_worker, name) if not ok: raise HTTPException(status_code=404, detail="no such worker") return {"removed": name} @@ -1669,7 +1693,7 @@ async def generator_send_claim_message(request: Request, body: dict = Body(...), @router.get("/admin/dash/generator/proxies") async def generator_proxies(request: Request, key: str = Depends(admin_key)) -> dict: - return _generator(request).checker.status() + return await asyncio.to_thread(_generator(request).checker.status) @router.post("/admin/dash/generator/proxies/run") @@ -1691,7 +1715,8 @@ def _flag(name): v = body.get(name) return None if v is None else bool(v) - return checker.run( + return await asyncio.to_thread( + checker.run, action, timeout=_num("timeout", int), concurrency=_num("concurrency", int), @@ -1707,7 +1732,7 @@ def _flag(name): @router.post("/admin/dash/generator/proxies/stop") async def generator_proxies_stop(request: Request, key: str = Depends(admin_key)) -> dict: - _generator(request).checker.stop() + await asyncio.to_thread(_generator(request).checker.stop) return {"ok": True} @@ -1752,14 +1777,14 @@ async def generator_proxies_maintain(request: Request, body: dict = Body(...), action = str(body.get("action") or "").strip() if action not in ("prune", "clear-bad", "clear-used", "clear-dead"): raise HTTPException(status_code=400, detail="unknown maintenance action") - return _generator(request).checker.run(action) + return await asyncio.to_thread(_generator(request).checker.run, action) @router.post("/admin/dash/generator/proxies/countries") async def generator_proxies_countries(request: Request, key: str = Depends(admin_key)) -> dict: """Look up the country of every pool proxy that has none (online, nothing is downloaded). Runs in the proxy checker so its log and progress show up.""" - return _generator(request).checker.run("countries") + return await asyncio.to_thread(_generator(request).checker.run, "countries") # ---- Edge Relays (Cloudflare, Vercel, Deno, Netlify) ------------------------ # diff --git a/zcode_gateway/api_health.py b/zcode_gateway/api_health.py index 21bacfa..82b27ab 100644 --- a/zcode_gateway/api_health.py +++ b/zcode_gateway/api_health.py @@ -4,11 +4,11 @@ import asyncio import os -import hmac from fastapi import APIRouter, Request, Response from fastapi.responses import JSONResponse +from .deps import _compare from .peer import peer_is_local router = APIRouter(tags=["ops"]) @@ -31,7 +31,7 @@ async def health(request: Request, response: Response) -> dict: nonce = os.environ.get("ZCODE_GATEWAY_UPDATE_NONCE", "") supplied = request.headers.get("X-AutoZCoder-Update", "") valid_nonce = bool(nonce and len(nonce) <= 128 and all(c in "0123456789abcdef" for c in nonce)) - update_probe = valid_nonce and hmac.compare_digest(nonce, supplied) + update_probe = valid_nonce and _compare(nonce, supplied) if not _ops_exposed(request) and not update_probe: return JSONResponse(status_code=404, content={"detail": "Not Found"}) state = request.app.state.app_state diff --git a/zcode_gateway/breaker.py b/zcode_gateway/breaker.py index 9fd3b6d..4654ec1 100644 --- a/zcode_gateway/breaker.py +++ b/zcode_gateway/breaker.py @@ -73,12 +73,17 @@ def note_edge_block(self, body: str | bytes | None = None) -> bool: if not _looks_like_edge_block(body): return False now = time.time() + # is_blocked() expires a stale episode (and resets the count) first, so + # `new_episode` is true when the previous block aged out: the next + # block after CLEAN_AFTER_S is a fresh episode and must re-fire the + # callback. + new_episode = not self.is_blocked() self.blocked_at = now self.consecutive += 1 log.warning( "edge block detected (HTML 403, #%d) — failing fast %ds, probing", self.consecutive, BLOCK_RETRY_AFTER_S) - if self.on_block is not None: + if self.on_block is not None and new_episode: try: self.on_block(self.consecutive) except Exception: diff --git a/zcode_gateway/captcha_tickets.py b/zcode_gateway/captcha_tickets.py index 2981f0d..c8dde23 100644 --- a/zcode_gateway/captcha_tickets.py +++ b/zcode_gateway/captcha_tickets.py @@ -141,8 +141,10 @@ def take(self) -> CaptchaTicket | None: with self._lock: live = [t for t in self._tickets if now - t.minted_at <= self.ttl - SAFETY_MARGIN] - self._tickets.clear() - ticket = live[-1] if live else None + # Only the handed-out ticket and the stale ones leave the pool: the + # other live tickets were already paid for and are still usable. + ticket = live.pop() if live else None + self._tickets[:] = live self._wake.set() return ticket diff --git a/zcode_gateway/generator.py b/zcode_gateway/generator.py index 00a95ce..5ae4303 100644 --- a/zcode_gateway/generator.py +++ b/zcode_gateway/generator.py @@ -3770,7 +3770,8 @@ def _auto_preflight(self, config: dict) -> None: def _pool_has_candidates(self) -> bool: try: - return any(line.strip() and not line.startswith("#") for line in self.pool_file.read_text().splitlines()) + return any(line.strip() and not line.startswith("#") for line in self.pool_file.read_text( + encoding="utf-8", errors="ignore").splitlines()) except OSError: return False diff --git a/zcode_gateway/gui.py b/zcode_gateway/gui.py index c251253..cc18ff7 100644 --- a/zcode_gateway/gui.py +++ b/zcode_gateway/gui.py @@ -19,6 +19,7 @@ import secrets import shutil import sys +import tempfile import threading import time import tkinter as tk @@ -346,6 +347,21 @@ def find_config() -> Path | None: return None +def _write_config_atomic(path: Path, text: str) -> None: + """Replace *path* atomically: a crash or full disk mid-write must never + truncate the live config holding the admin and client keys.""" + fd, name = tempfile.mkstemp(prefix=path.name + ".", suffix=".tmp", dir=path.parent) + tmp = Path(name) + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + stream.write(text) + stream.flush() + os.fsync(stream.fileno()) + os.replace(tmp, path) + finally: + tmp.unlink(missing_ok=True) + + def _normalize_config(cfg_path: Path | None) -> None: """First-run repair: point the config at a real accounts folder. @@ -388,8 +404,7 @@ def _normalize_config(cfg_path: Path | None) -> None: except OSError: pass try: - cfg_path.write_text(yaml.safe_dump(data, sort_keys=False), - encoding="utf-8") + _write_config_atomic(cfg_path, yaml.safe_dump(data, sort_keys=False)) except OSError: pass @@ -421,6 +436,8 @@ class ServerController: def __init__(self) -> None: self._server = None self._thread: threading.Thread | None = None + self._config_ready = threading.Event() + self._stop_requested = threading.Event() self.host = "127.0.0.1" self.port = 8788 self.error: str | None = None @@ -439,6 +456,8 @@ def start(self, cfg_path: Path | None) -> None: if self.running: return self.error = None + self._config_ready.clear() + self._stop_requested.clear() def run() -> None: try: @@ -446,8 +465,12 @@ def run() -> None: from zcode_gateway.app import create_app from zcode_gateway.config import load_config + if self._stop_requested.is_set(): + return + cfg = load_config(str(cfg_path) if cfg_path else None) self.host, self.port = cfg.server.host, cfg.server.port + self._config_ready.set() # If the configured accounts folder is not there (e.g. a default # config next to a copied exe), fall back to the folder we found. @@ -468,6 +491,8 @@ def run() -> None: self.accounts_loaded = 0 app = create_app(cfg) + if self._stop_requested.is_set(): + return # log_config=None stops uvicorn installing its own stdout-based # handlers. In a --windowed frozen build sys.stdout can still be # None, and uvicorn's default config would then raise on a @@ -485,6 +510,10 @@ def run() -> None: self._thread.start() def stop(self, wait: float = 5.0) -> None: + # run() may still be in its imports/create_app stretch with self._server + # not yet assigned; this flag makes it exit before binding rather than + # bring up a gateway the UI has already reported as stopped. + self._stop_requested.set() server = self._server if server is not None: server.should_exit = True @@ -513,6 +542,11 @@ def wait_ready(self, timeout: float = 25.0) -> bool: while time.time() < deadline: if self.error: return False + if not self._config_ready.is_set(): + # host/port still hold the defaults; a foreign process that + # answers /health there is not the gateway we started + time.sleep(0.05) + continue try: with urllib.request.urlopen(self.base_url + "/health", timeout=1): return True @@ -1186,8 +1220,8 @@ def _import_source_mismatch(self, result) -> None: import yaml data = yaml.safe_load(self.cfg_path.read_text(encoding="utf-8")) or {} (data.setdefault("accounts", {}))["source"] = "accounts_dir" - self.cfg_path.write_text(yaml.safe_dump(data, sort_keys=False), - encoding="utf-8") + _write_config_atomic(self.cfg_path, + yaml.safe_dump(data, sort_keys=False)) self.restart_server(f"Imported {result['written']} account(s); gateway restarted with accounts_dir") except Exception as exc: self.toast(f"could not update config: {exc}", "err") @@ -1356,8 +1390,8 @@ def act_regen_keys(self): keys = server.get("client_keys") if isinstance(keys, list) and keys and isinstance(keys[0], dict): keys[0]["key"] = f"gw-{secrets.token_hex(8)}" - self.cfg_path.write_text(yaml.safe_dump(data, sort_keys=False), - encoding="utf-8") + _write_config_atomic(self.cfg_path, + yaml.safe_dump(data, sort_keys=False)) self.accounts_dir = self._configured_accounts_dir() except Exception as exc: return self.toast(f"could not update config: {exc}", "err") @@ -1522,8 +1556,8 @@ def act_save_settings(self): fallback = data.setdefault("models", {}).setdefault("auto_free_fallback", {}) fallback["enabled"] = bool(self.free_fallback_enabled.get()) fallback["chain"] = fallback_chain - self.cfg_path.write_text(yaml.safe_dump(data, sort_keys=False), - encoding="utf-8") + _write_config_atomic(self.cfg_path, + yaml.safe_dump(data, sort_keys=False)) except Exception as exc: return self.toast(str(exc), "err") @@ -1705,6 +1739,11 @@ def _looks_like_port_conflict(err: str) -> bool: def stop_everything(self) -> None: self._lifecycle += 1 + # a user Stop always wins over a restart still waiting for the old + # server to release the port: bumping the generation above makes its + # queued _start_after_restart a no-op + self._restart_pending = False + self._restart_message = None self._stopping = True for var in getattr(self, "server_urls", {}).values(): var.set("") @@ -1720,24 +1759,29 @@ def stop_everything(self) -> None: def restart_server(self, success_message: str | None = None) -> None: if self._restart_pending: return + self.stop_everything() self._restart_pending = True self._restart_message = success_message - self.stop_everything() + generation = self._lifecycle self._stopping = True def wait_and_start() -> None: if self.server.wait_stopped(): - self.after(0, self._start_after_restart) + self.after(0, self._start_after_restart, generation) else: - self.after(0, self._restart_failed) + self.after(0, self._restart_failed, generation) threading.Thread(target=wait_and_start, daemon=True).start() - def _start_after_restart(self) -> None: + def _start_after_restart(self, generation: int | None = None) -> None: + if generation is not None and generation != self._lifecycle: + return if not self._restart_pending: return self._stopping = False self.start_everything() - def _restart_failed(self) -> None: + def _restart_failed(self, generation: int | None = None) -> None: + if generation is not None and generation != self._lifecycle: + return self._restart_pending = False self._restart_message = None self.toast("gateway did not stop cleanly; restart aborted", "err") diff --git a/zcode_gateway/notify.py b/zcode_gateway/notify.py index 24673d7..ef84d42 100644 --- a/zcode_gateway/notify.py +++ b/zcode_gateway/notify.py @@ -57,7 +57,10 @@ def emit(self, category: str, message: str, level: str = "info", now = time.time() for e in reversed(self._events): if now - e["ts"] > window: - break + # a matching entry bumped in place keeps its old + # deque position, so ts is not monotonic: skip the + # stale entry and keep scanning + continue if (e["category"], e["message"]) == key: e["count"] += 1 e["ts"] = now diff --git a/zcode_gateway/pool.py b/zcode_gateway/pool.py index b1309e1..9812a27 100644 --- a/zcode_gateway/pool.py +++ b/zcode_gateway/pool.py @@ -842,7 +842,10 @@ async def cool(self, account: Account, seconds: int | None = None, if expected_credentials is not None and (self._accounts.get(account.id) is not account or account.credential_key() != expected_credentials): return - if account.state is AccountState.DEAD: + # EXHAUSTED is quota-drained retirement, not a transient error: a + # COOLING downgrade puts the account back in rotation once the + # cooldown lapses, so it is as sticky as DEAD here. + if account.state in (AccountState.DEAD, AccountState.EXHAUSTED): return account.state = AccountState.COOLING account.cooldown_until = time.time() + (seconds or self.cfg.cooldown_seconds) @@ -909,6 +912,12 @@ async def kill_after_strikes(self, account: Account, reason: str = "", account.death_reason = reason or 'authentication_error' account.last_error = reason return + if account.state is AccountState.EXHAUSTED: + # Sticky like DEAD: the account is retired for quota, not for + # this request's auth failure, and cooling it would re-enter + # rotation when the cooldown lapsed. + account.last_error = reason + return account.auth_strikes = n if n >= strikes: account.state = AccountState.DEAD @@ -945,6 +954,10 @@ async def enable(self, account_id: str) -> bool: acc.cooldown_until = 0.0 acc.error_count = 0 acc.last_error = "" + # an explicit re-enable is a fresh strike budget: banked strikes + # would otherwise retire the account on its first future failure + acc.auth_strikes = 0 + acc.quota_strikes = 0 return True async def upsert(self, account: Account) -> None: @@ -966,15 +979,16 @@ async def upsert(self, account: Account) -> None: existing.expires_at = account.expires_at existing.credential_generation += 1 existing.email = account.email or existing.email - existing.state = account.state if account.state is not AccountState.DEAD else existing.state - if existing.state is not AccountState.DEAD: - existing.death_reason = '' existing.paid_exhausted = account.paid_exhausted if account.notes: existing.notes.update(account.notes) - # a re-login is a fresh credential: clear auth strikes but keep - # model caps (they persist on disk and are model-scoped facts) + # a re-login is a fresh credential: clear both strike budgets but + # keep model caps (they persist on disk and are model-scoped facts). + # Without the quota reset, an account that recovered from + # quota-strike EXHAUSTION re-retired on its first future rejection + # with the old strikes already banked. existing.auth_strikes = 0 + existing.quota_strikes = 0 async def remove(self, account_id: str) -> bool: async with self._lock: diff --git a/zcode_gateway/relay_manager.py b/zcode_gateway/relay_manager.py index 42ee86c..72c5efe 100644 --- a/zcode_gateway/relay_manager.py +++ b/zcode_gateway/relay_manager.py @@ -15,6 +15,7 @@ import json import copy +import logging import os import random import threading @@ -31,6 +32,8 @@ ping_relay, ) +log = logging.getLogger("zcode_gateway.relay_manager") + STRATEGIES = ("single", "round_robin", "random") FAIL_COOLDOWN_S = 60.0 # a failed relay sits out one minute @@ -131,13 +134,17 @@ def get_settings(self) -> dict[str, Any]: if self.path.exists(): try: data = json.loads(self.path.read_text(encoding="utf-8")) - if isinstance(data, dict): - settings = self._default_settings() - settings.update(data) - self._cache = settings - return copy.deepcopy(settings) - except Exception: - pass + except Exception as exc: + # Never cache defaults for an unreadable file: a later save + # would write them over the user's relays for good. + log.warning("relays.json could not be read (%s) — serving " + "defaults for this read only", exc) + return self._default_settings() + if isinstance(data, dict): + settings = self._default_settings() + settings.update(data) + self._cache = settings + return copy.deepcopy(settings) settings = self._default_settings() self._cache = settings return copy.deepcopy(settings) @@ -378,6 +385,8 @@ def update_options(self, models: list[str] | None = None, with self._lock: st = self.get_settings() if models is not None: + if isinstance(models, str): # a single model name, not a list + models = [models] st["models"] = list(dict.fromkeys(models)) if enabled is not None: st["enabled"] = bool(enabled) diff --git a/zcode_gateway/service.py b/zcode_gateway/service.py index 45212db..d8ab4cb 100644 --- a/zcode_gateway/service.py +++ b/zcode_gateway/service.py @@ -740,10 +740,15 @@ def _record(self, account, headers, body, status, response_headers, response, du duration_ms=duration, prompt_tokens=converted['prompt_tokens'], completion_tokens=converted['completion_tokens'], total_tokens=converted['total_tokens'], error_code=None if status == 200 else f'http_{status}') - cap = dict(request_headers=dict(headers), request_body=copy.deepcopy(body), status=status, - response_headers=dict(response_headers), response_body=response, - duration_ms=duration, account_id=account.id, dialect=dialect, - model=model, variant='anthropic') + # Building the capture payload deep-copies the request body; on the + # event loop that stalls every stream, and it is pure waste when the + # JSONL capture is off. The usage row is written either way. + cap = None + if getattr(self.capture, 'enabled', True): + cap = dict(request_headers=dict(headers), request_body=copy.deepcopy(body), status=status, + response_headers=dict(response_headers), response_body=response, + duration_ms=duration, account_id=account.id, dialect=dialect, + model=model, variant='anthropic') # Disk I/O off the event loop: SQLite/JSONL writes (and AV scans on # those files) used to stall every concurrent stream while they ran. try: @@ -766,6 +771,7 @@ async def aclose(self): def _write_record(self, rec, cap): try: self.store.record(**rec) - self.capture.write(**cap) + if cap is not None: + self.capture.write(**cap) except Exception: log.debug("store/capture write failed", exc_info=True) diff --git a/zcode_gateway/tokens.py b/zcode_gateway/tokens.py index 5422247..da8ef09 100644 --- a/zcode_gateway/tokens.py +++ b/zcode_gateway/tokens.py @@ -163,7 +163,10 @@ def _apply_credentials_locked(self, account, *, access_token, refresh_token=None account.last_error = '' account.state = AccountState.READY account.death_reason = '' + # a replaced credential gets a fresh strike budget: banked quota + # strikes would re-retire it on the first future rejection account.auth_strikes = 0 + account.quota_strikes = 0 account.notes.pop('quota_check_error', None) async def apply_credentials(self, account, **credentials): diff --git a/zcode_gateway/translate_anthropic.py b/zcode_gateway/translate_anthropic.py index d74fcf7..faf1386 100644 --- a/zcode_gateway/translate_anthropic.py +++ b/zcode_gateway/translate_anthropic.py @@ -11,6 +11,8 @@ import uuid from typing import Any, Iterable +from .upstream import split_sse_lines + # --------------------------------------------------------------------------- # # request: anthropic -> openai # --------------------------------------------------------------------------- # @@ -476,7 +478,7 @@ def extract_stream_error(text: str) -> dict | None: """ import json as _json - for line in (text or "").splitlines(): + for line in split_sse_lines(text): line = line.strip() if not line.startswith("data:"): continue @@ -503,7 +505,7 @@ def stream_had_content(text: str) -> bool: """True if the stream carried any assistant text or tool call.""" import json as _json - for line in (text or "").splitlines(): + for line in split_sse_lines(text): line = line.strip() if not line.startswith("data:") or "[DONE]" in line: continue diff --git a/zcode_gateway/upstream.py b/zcode_gateway/upstream.py index d3394da..5b780fa 100644 --- a/zcode_gateway/upstream.py +++ b/zcode_gateway/upstream.py @@ -9,6 +9,7 @@ import asyncio import json import logging +import re import time import uuid from dataclasses import dataclass @@ -373,10 +374,10 @@ async def send(self, headers: dict[str, str], body: dict[str, Any], relay_url = ((relay_entry or {}).get("url") or "").strip() or None relay_secret = (relay_entry or {}).get("secret") or "" if relay_url and relay_url in tried_relay_urls: - # already tried this one this call — stop rotating - relay_url = None if len(tried_relay_urls) >= self.relay_manager.get_relay_count() else relay_url - if relay_url is None: - break + # this relay already failed this call — stop rotating + # rather than send to it a second time; the last relay + # error is re-raised below + break try: if relay_url: @@ -503,13 +504,26 @@ class UpstreamBlockThroughRelay(TransportError): # --------------------------------------------------------------------------- # +_SSE_LINE_BREAK = re.compile(r"\r\n|\r|\n") + + +def split_sse_lines(text: str) -> list[str]: + """Split an SSE block into lines on real line terminators only. + + str.splitlines() also breaks on U+2028, U+2029 and U+0085, which a JSON + payload written with ensure_ascii=False carries raw inside string values. + One such character used to cut the data line mid-string and fail parsing. + """ + return _SSE_LINE_BREAK.split(text or "") + + def iter_sse_events(chunk_text): events = [] for frame in __import__('re').split(r'\r\n\r\n|\n\n|\r\r', chunk_text): # SSE spec: strip the field name and EXACTLY one optional leading # space — lstrip(' ') ate multi-space-prefixed payloads lines = [line[5:][1:] if line[5:].startswith(' ') else line[5:] - for line in frame.splitlines() if line.startswith('data:')] + for line in split_sse_lines(frame) if line.startswith('data:')] if not lines: continue data = '\n'.join(lines) if not data.strip(): continue