diff --git a/ai_http.py b/ai_http.py index c1ea21bf..d1ac25ac 100644 --- a/ai_http.py +++ b/ai_http.py @@ -88,6 +88,26 @@ def collect(value, depth=0): return 'Provider detail: ' + ' | '.join(messages)[:340] if messages else 'The provider returned no readable error message.' +def pumas_error_detail(data, payload=None, headers=None): + """Pumas' OpenAI-compatible errors carry its (redacted) reason in `error.message`. + + Returns '' for any body that is not a bounded `pumas_error`; that reason is + still passed through the same redaction as other provider diagnostics. + """ + if not data or len(data) > MAX_ERROR_BODY: + return '' + try: + value = json.loads(data) + except (ValueError, UnicodeError): + return '' + error = value.get('error') if isinstance(value, dict) else None + if (not isinstance(error, dict) or error.get('type') != 'pumas_error' + or not isinstance(error.get('message'), str) or not error['message'].strip()): + return '' + detail = provider_error_detail(json.dumps({'message': error['message']}).encode(), payload, headers) + return 'Pumas reported: ' + detail.removeprefix('Provider detail: ') if detail.startswith('Provider detail: ') else '' + + def request_error_hint(data): """Classify bounded provider errors without exposing arbitrary response text.""" try: @@ -184,18 +204,24 @@ def watch_cancel(): 403: 'Check account permissions and provider restrictions.', 429: 'Rate limit reached; wait before trying again.'}.get(response.status, 'Check the selected model and structured-output support.') - if response.status == 400 or response.status >= 500: - try: - transport.settimeout(max(.001, min(2, deadline - time.monotonic()))) - error_data = response.read(MAX_ERROR_BODY + 1) - specific = request_error_hint(error_data) if len(error_data) <= MAX_ERROR_BODY else '' + try: + transport.settimeout(max(.001, min(2, deadline - time.monotonic()))) + error_data = response.read(MAX_ERROR_BODY + 1) + except (OSError, http.client.HTTPException): + error_data = None + if label == 'OpenRouter' and (response.status == 400 or response.status >= 500): + hint = 'Could not read the provider error body within the diagnostic limit.' + if error_data is not None: + specific = request_error_hint(error_data) if len(error_data) <= MAX_ERROR_BODY else '' + pumas_detail = pumas_error_detail(error_data, payload, headers) + if pumas_detail: + # Pumas' own (redacted) reason is shown for every refusal status. + hint = (specific + ' ' if specific else '') + pumas_detail + elif response.status == 400 or response.status >= 500: if label == 'OpenRouter' or response.status >= 500: hint = provider_error_detail(error_data, payload, headers) elif specific: hint = specific - except (OSError, http.client.HTTPException): - if label == 'OpenRouter': - hint = 'Could not read the provider error body within the diagnostic limit.' raise CodexError(f'{label} returned HTTP {response.status}. {hint}', retryable=response.status in (502, 503, 504)) yield response, transport, deadline diff --git a/docs/contracts/pumas-image-to-text-consumer.md b/docs/contracts/pumas-image-to-text-consumer.md index a6fa3697..c557e28e 100644 --- a/docs/contracts/pumas-image-to-text-consumer.md +++ b/docs/contracts/pumas-image-to-text-consumer.md @@ -1,22 +1,52 @@ # Pumas Image→Text consumer -This opt-in caption path consumes the contract at Pumas Library merged PR68, -source `36c2aab2c885b8416fd4e39cf8c37c510596d6e4`: -[Image→Text contract](https://github.com/MrScripty/Pumas-Library/blob/36c2aab2c885b8416fd4e39cf8c37c510596d6e4/docs/contracts/image-to-text-v0.8.md). -It is based on Tuldok main `d99e23d2b0d2783a7a9863232a7453cd9f2db7aa`. -The legacy compatible caption path and the previous typed text/image operations -remain available. Their existing producer receipt is not silently repinned. +This opt-in caption path, and the typed text and image operations that share +`pumas_operations`, consume the Pumas Library v0.8 typed HTTP contract on branch +`feat/v080-consumer-api`, source `4c79cd8e35db566b70caaa13d8d8f92265f5976e`: +[model operations, contract version 1](https://github.com/MrScripty/Pumas-Library/blob/4c79cd8e35db566b70caaa13d8d8f92265f5976e/docs/contracts/model-operations-v1.md). +It supersedes the merged PR68 pin `36c2aab2c885b8416fd4e39cf8c37c510596d6e4`, +which `source.json` keeps as `previous_producer`. The legacy compatible caption +path remains available. The PR54 typed text/image capture receipt in +`tests/fixtures/pumas-typed-v1` is historical and is not repinned. -`pumas_operations.validate_manifest` accepts all seven closed capability variants, -the three additional image input formats and image byte/pixel/count option names. -The existing unavailable reasons remain unchanged. Availability is a live -observation, not a model qualification receipt. Audio's existing -`max_output_tokens` descriptor option is accepted as declared in the inspected DTO. +### Additive compatibility + +Pumas responses grow additively within contract version 1. Tuldok validates +only what it uses and selects capabilities and option bounds by name, never by +position or count: + +- `GET /v1/capabilities` requires `supported_contract_versions` containing 1, + `model`, an exact `profile`, positive `max_request_bytes`/`max_response_bytes` + and a `capabilities` list. Other fields (`context_tokens`, + `max_stream_event_bytes`, future fields) are retained but not required. +- Only `chat_generation`, `image_generation` and `image_to_text` descriptors are + validated: fixed semantic task, string format lists, an availability `state` + with an optional string reason, and finite, ordered bounds for the options + Tuldok compares (`max_tokens`, `temperature`, `top_p`, `width`, `height`, + `seed`, `image_bytes`, `image_pixels`, `image_count`). A used capability or + option advertised twice is refused as ambiguous. +- Other capabilities (`multimodal_embedding`, `decision`, audio, future values), + formats (`content_batch`, `decision`, `decisions`, future values), option names, + availability states and reasons, and response or result fields are ignored. + Any availability other than `available` means unavailable, and its reason is + shown. +- The capability observation is bounded at 1 MiB. `GET /v1/served-models` is + preferred for listing serving aliases with their profiles; gateways without it + (HTTP 404) fall back to `/v1/models`, which lists each ID once with `profiles`. + +Availability is a live observation, not a model qualification receipt. Choose **Pumas typed v1** in the image caption form, select the serving alias and optional exact profile, and inspect capabilities. On each new attempt, the owner requires `pumas.model-operations.image-to-text:1` in the gateway build advertisement -and an available Image→Text descriptor for the exact resolved model/profile. +and an available Image→Text descriptor for the exact resolved model/profile. The +gateway advertisement (`GET /.well-known/pumas`) has its own versioned decoder, +which also grows additively: unknown fields and new protocol, schema, capability or +compiled-feature entries are ignored and are not retained in the stored build +observation, which keeps only the known fields. Missing required fields, wrong +types, unsupported schema versions and malformed endpoints are refused with a +reason naming the field; a gateway scan lists such refused advertisements with +their reasons instead of hiding them. The request uses the facade's `messages`, `semantic_task:"image_to_text"`, text-generation options and `stream:false`. The seed is omitted. Named image and `image_messages` inputs are also supported by the bounded consumer request builder. @@ -25,8 +55,10 @@ messages can contain images; audio, URLs, paths and data-URL envelopes are refus The consumer applies the producer's 32 MiB whole-request, 8 MiB per-image, 16 MiB aggregate, four-image, 4096-axis, 4,194,304-pixel, 128-message and 128-total-part -bounds. Tokens default to 512 and permit 1–2048; temperature/top-p must be finite -and within 0–2/0–1. The caption application's existing preparation is smaller: +bounds. Tokens default to 512. Pumas no longer caps Image→Text `max_tokens` at +2048; the live `max_tokens` bound follows the served context window and is +checked before admission. Temperature/top-p must be finite and within 0–2/0–1. +Advertised image byte, pixel, count and dimension bounds are checked when present. The caption application's existing preparation is smaller: one RGB JPEG, at most 2 MiB and 1600 pixels on its longest side. It decodes the captured, hash-verified canonical image and retains both canonical-source and prepared-input hashes. Preparation is explicit in request history. @@ -37,8 +69,8 @@ replace Pumas's production PNG/turbojpeg validation, allocation controls or admission authority, and does not claim equivalent JPEG warning detection. No local validation result is used as permission to bypass provider refusals. -The response is the closed `{contract_version, request_id, result}` envelope. -The request ID must match; `result.kind` must be text. There is no response-model +The response is the `{contract_version, request_id, result}` envelope; added +fields are ignored. The request ID must match; `result.kind` must be text. There is no response-model requirement and no private provider envelope projection. This application requires a complete `stop` finish reason and valid caption JSON within a 256 KiB response bound; partial or filtered output is not applied. Capability/build observations, @@ -47,8 +79,15 @@ on the exact job. List responses omit image-bearing canonical requests and raw bytes. Explicit application creates a draft target and records its evidence in revision history; human review remains separate. -HTTP 400/413/422/503 pre-admission refusals and 502 failures retain their typed -code, HTTP status and `outcome` in the asynchronous job. A disconnect, uncertain +Every typed error is `{contract_version, request_id, error:{code, outcome, message}}`. +`error.outcome` is authoritative: `not_admitted` records no provider effect and +anything else is `unknown`, whatever the HTTP status. 404 `model_not_found` and +409 `ambiguous_model`/`ambiguous_operation` are ordinary refusals. An error whose +`request_id` is `null` (Pumas could not read the body) is still a typed refusal. +Unknown error codes are kept verbatim. The job retains the code, HTTP status and +outcome, and its error text includes Pumas' `error.message`, bounded and on one +line. Capability refusals show the same message. On the OpenAI-compatible routes, +a `pumas_error` body's `error.message` is shown after the existing redaction. A disconnect, uncertain response or possibly dispatched interrupted attempt remains `unknown`. Repeating an existing exact application request ID returns the retained attempt; it never replays inference. A new attempt requires a new explicit request. Browser recovery @@ -65,27 +104,30 @@ and packaged application acceptance remain integration-lead/provider-owner gates `tests/fixtures/pumas-image-to-text-v1/source.json` labels the capability/wire fixture as **source-derived**, with inspected production-source hashes. It is not -a captured native DTO schema export or an actual model reply. The tests exercise -seven-capability parsing, named/facade PNG/JPEG inputs (including progressive +a captured native DTO schema export or an actual model reply. The fixture follows +the pinned descriptor construction for a llama.cpp vision serving with an +8192-token context, including the `multimodal_embedding` and `decision` +descriptors. The tests exercise additive capability parsing, named/facade PNG/JPEG inputs (including progressive JPEG), order, closed shapes, finite/resource bounds, typed application captions, draft/history, live selection refusal, cancellation/borrowed-owner survival, HTTP outcomes, restart and no replay. The existing typed browser suite additionally checks the actual caption form, capability inspection, application and disconnect. -The six actual production DTO schemas regenerated from the combined source -are retained in `tests/fixtures/pumas-image-to-text-v1/schemas/`, with SHA-256 -identities in `source.json`. The source-derived capability fixture and authored -named/facade image requests and typed responses validate against these exports. -The schemas describe wire shape; finite bounds and live readiness remain runtime -checks. Neither fixture output nor schema validity proves native inference. +The six actual production DTO schemas regenerated at the pinned source with the +`export-contract` feature are retained in +`tests/fixtures/pumas-image-to-text-v1/schemas/`, with SHA-256 identities in +`source.json`. All six changed from the PR68 export (new capabilities, formats, +`context_tokens`, decision and content-batch DTOs and the required error +`message`). The +source-derived capability fixture and authored named/facade image requests and +typed responses validate against these exports in the vision tests. The schemas +describe wire shape; finite bounds and live readiness remain runtime checks. +Neither fixture output nor schema validity proves native inference. -The reviewed producer retains identical DTO definitions and schema dependencies; -all six exported schema SHA-256 identities are unchanged. `source.json` records -the export commit and refreshed handler/controlled-test source hashes. The fixture -gateway advertisement reads the same versioned source pin as its consumer -acceptance evidence. -This pin is the immutable merged Pumas main commit. Its source tree is identical -to the reviewed producer that passed CI before the coordinated merge. +`source.json` records the branch, commit, tree, contract document hash and +handler/controlled-test source hashes. The pin is a branch commit, not a merged +Pumas main commit; repin when it merges. The fixture gateway advertisement reads +the same versioned source pin as its consumer acceptance evidence. The combined desktop contract and packaged application are provider integration gates. Real model/projector inference, semantic quality, managed native teardown and production audio qualification remain unclaimed. No models or runtimes are diff --git a/gateway_discovery.py b/gateway_discovery.py index 314023d3..7528c658 100644 --- a/gateway_discovery.py +++ b/gateway_discovery.py @@ -35,7 +35,10 @@ def listening_endpoints(): return sorted(endpoints, key=lambda endpoint: (endpoint[1], endpoint[0])) -def get_json(host, port, path, deadline, payload=None, max_response=MAX_RESPONSE, decode=json.loads, require_complete=False): +def get_json(host, port, path, deadline, payload=None, max_response=MAX_RESPONSE, decode=json.loads, require_complete=False, + error=None): + """GET/POST bounded JSON. With `error`, a non-200 body is read under the same bounds + and passed to `error(status, raw)`, which raises a readable refusal.""" timeout = min(1.5, deadline - time.monotonic()) if timeout <= 0: raise TimeoutError() @@ -64,7 +67,7 @@ def expire(): expiry.daemon = True expiry.start() with connection.getresponse() as response: - if response.status != 200: + if response.status != 200 and error is None: raise ValueError('Not a gateway response') if require_complete: declared = response.getheader('Content-Length') @@ -86,6 +89,9 @@ def expire(): raise ValueError('Pumas descriptor response ended before its declared length') if require_complete and time.monotonic() >= deadline: raise TimeoutError() + if response.status != 200: + error(response.status, bytes(data)) + raise ValueError('Not a gateway response') return decode(data) finally: if expiry is not None: @@ -212,15 +218,20 @@ def scan_advertised(): def observe(item): host, port = item address = '[' + host + ']' if ':' in host else host + url = f'http://{address}:{port}' try: - return inspect_advertised(f'http://{address}:{port}', deadline) + return inspect_advertised(url, deadline), None + except descriptor.Rejected as error: + # Pumas answered but its advertisement was refused: keep the reason for the user. + return None, {'server_url': url, 'reason': str(error)} except (OSError, ValueError, TypeError, http.client.HTTPException): - return None + return None, None with ThreadPoolExecutor(max_workers=32) as pool: results = list(pool.map(observe, endpoints)) # Distinct IPv4/IPv6 services on one port must retain their library context. - found = {row['server_url']: row for row in results if row is not None} - return {'gateways': list(found.values()), 'qualification': + found = {row['server_url']: row for row, _ in results if row is not None} + rejected = [refusal for _, refusal in results if refusal is not None] + return {'gateways': list(found.values()), 'rejected': rejected, 'qualification': 'Read-only HTTP advertisements. Core ownership is not authenticated; compiled features do not prove model readiness.'} finally: SCAN_LOCK.release() diff --git a/image_generation.py b/image_generation.py index 76e37f2f..2ca126f5 100644 --- a/image_generation.py +++ b/image_generation.py @@ -258,7 +258,13 @@ def watch_cancel(): 499: 'Image generation cancelled.', 503: 'The image model is unavailable. Load it in Pumas and refresh models.', 507: 'Pumas reported insufficient GPU memory. Free memory before trying again.'} - raise ValueError(messages.get(response.status, 'Pumas image generation failed. Check its runtime status. The request was not retried.')) + message = messages.get(response.status, 'Pumas image generation failed. Check its runtime status. The request was not retried.') + try: + transport.settimeout(2) + detail = ai_http.pumas_error_detail(response.read(ai_http.MAX_ERROR_BODY + 1), payload) + except (OSError, http.client.HTTPException): + detail = '' + raise ValueError(message + (' ' + detail if detail else '')) data = bytearray() while not response.isclosed(): chunk = response.read1(65536) diff --git a/pumas_gateway_descriptor.py b/pumas_gateway_descriptor.py index eb25b9f1..b1eb3ec0 100644 --- a/pumas_gateway_descriptor.py +++ b/pumas_gateway_descriptor.py @@ -1,12 +1,23 @@ -"""Decode the public Pumas PR51 HTTP advertisement; no owner authentication.""" +"""Decode the public Pumas HTTP advertisement (`GET /.well-known/pumas`); no owner authentication. + +The advertisement grows additively. Unknown fields and new entries in open lists +(protocols, schemas, capabilities, compiled features) are ignored: they are neither +errors nor retained in the decoded advertisement. Missing required fields, wrong +types, unsupported schema versions and malformed endpoints are refused with a +reason naming the field. +""" import hashlib import ipaddress import json from urllib.parse import urlsplit SOURCE_COMMIT = '80f06ab17f9eea639fee143c86319ca9b5e1a21c' +# Transport bound. Open lists have no entry-count cap of their own. MAX_DESCRIPTION_BYTES = 64 * 1024 -MAX_ITEMS = 64 + + +class Rejected(ValueError): + """A Pumas advertisement that was read and refused. The message is the reason.""" def endpoint(value): @@ -28,60 +39,82 @@ def endpoint(value): return f'http://{address}:{port}' -def _text(value, limit=4096): +def _fail(where, problem): + raise Rejected(' '.join(filter(None, ('Pumas advertisement', where, problem))) + '.') + + +def _text(value, where, limit=4096): if not isinstance(value, str) or not value or len(value) > limit: - raise ValueError('Invalid Pumas descriptor text.') - value.encode('utf-8', errors='strict') + _fail(where, f'must be a nonempty string of at most {limit} characters') + try: + value.encode('utf-8', errors='strict') + except UnicodeError: + _fail(where, 'must be valid Unicode text') -def _uint(value): +def _uint(value, where): if type(value) is not int or not 0 <= value <= 4294967295: - raise ValueError('Invalid Pumas descriptor integer.') + _fail(where, 'must be an unsigned 32-bit integer') -def _object(value, required, optional=()): - if type(value) is not dict or not set(required) <= value.keys() or value.keys() - set(required) - set(optional): - raise ValueError('Unsupported Pumas descriptor fields.') +def _version(value, where): + if type(value) is not int: + _fail(where, 'must be an integer') + if value != 1: + raise Rejected(f'Pumas advertisement {where} {value} is unsupported; Tuldok supports version 1.') -def _list(value): - if type(value) is not list or len(value) > MAX_ITEMS: - raise ValueError('Invalid Pumas descriptor list.') +def _object(value, where, required, optional=()): + """Open shape: every required field present. Returns only the known fields.""" + if type(value) is not dict: + _fail(where, 'must be an object') + missing = [name for name in required if name not in value] + if missing: + _fail(where, 'is missing required field(s) ' + ', '.join(missing)) + known = set(required) | set(optional) + return {name: item for name, item in value.items() if name in known} -def _strings(value): - _list(value) - for item in value: - _text(item, 512) +def _list(value, where): + if type(value) is not list: + _fail(where, 'must be a list') + return value -def _protocols(value): - _list(value) - for item in value: - _object(item, ('name', 'versions')) - _text(item['name'], 512) - _list(item['versions']) - for version in item['versions']: - _uint(version) +def _strings(value, where): + for index, item in enumerate(_list(value, where)): + _text(item, f'{where}[{index}]', 512) + return list(value) -def _build(value): - _object(value, ('build_info_schema_version', 'component', 'package_version', 'compiled_features', 'protocols', 'schemas'), - ('build_id', 'source_revision', 'target')) - if type(value['build_info_schema_version']) is not int or value['build_info_schema_version'] != 1: - raise ValueError('Unsupported Pumas build-info schema.') +def _protocols(value, where): + result = [] + for index, item in enumerate(_list(value, where)): + item = _object(item, f'{where}[{index}]', ('name', 'versions')) + _text(item['name'], f'{where}[{index}].name', 512) + for position, version in enumerate(_list(item['versions'], f'{where}[{index}].versions')): + _uint(version, f'{where}[{index}].versions[{position}]') + result.append(item) + return result + + +def _build(value, where): + value = _object(value, where, ('build_info_schema_version', 'component', 'package_version', 'compiled_features', + 'protocols', 'schemas'), ('build_id', 'source_revision', 'target')) + _version(value['build_info_schema_version'], where + '.build_info_schema_version') for name in ('component', 'package_version', 'build_id', 'source_revision', 'target'): - if name in value and value[name] is not None: - _text(value[name]) - elif name in ('component', 'package_version'): - raise ValueError('Missing Pumas build identity.') - _strings(value['compiled_features']) - _protocols(value['protocols']) - _list(value['schemas']) - for item in value['schemas']: - _object(item, ('name', 'version')) - _text(item['name'], 512) - _uint(item['version']) + if name in ('component', 'package_version') or value.get(name) is not None: + _text(value[name], f'{where}.{name}') + value['compiled_features'] = _strings(value['compiled_features'], where + '.compiled_features') + value['protocols'] = _protocols(value['protocols'], where + '.protocols') + schemas = [] + for index, item in enumerate(_list(value['schemas'], where + '.schemas')): + item = _object(item, f'{where}.schemas[{index}]', ('name', 'version')) + _text(item['name'], f'{where}.schemas[{index}].name', 512) + _uint(item['version'], f'{where}.schemas[{index}].version') + schemas.append(item) + value['schemas'] = schemas + return value def _pairs(pairs): @@ -102,29 +135,34 @@ def decode(raw): raise ValueError('Pumas descriptor exceeds the 64 KiB observation limit.') try: value = json.loads(raw.decode('utf-8'), object_pairs_hook=_pairs, parse_constant=_constant) - _object(value, ('advertisement_schema_version', 'service_generation', 'instance', 'endpoint', 'build_info')) - if type(value['advertisement_schema_version']) is not int or value['advertisement_schema_version'] != 1: - raise ValueError('Unsupported Pumas HTTP advertisement schema.') - _text(value['service_generation'], 128) - endpoint(value['endpoint']) - instance = value['instance'] - _object(instance, ('discovery_schema_version', 'registry_library_id', 'library_root', 'generation', 'pumas_version', - 'protocols', 'capabilities', 'model_ref_schema_version', 'selector_schema_version'), ('build_info',)) - if type(instance['discovery_schema_version']) is not int or instance['discovery_schema_version'] != 1: - raise ValueError('Unsupported Pumas discovery schema.') - for name in ('registry_library_id', 'library_root', 'generation', 'pumas_version'): - _text(instance[name]) - for name in ('model_ref_schema_version', 'selector_schema_version'): - _uint(instance[name]) - _protocols(instance['protocols']) - _strings(instance['capabilities']) - if instance.get('build_info') is not None: - _build(instance['build_info']) - build = value['build_info'] - _build(build) - if (not any(p['name'] == 'pumas.local-http' and 1 in p['versions'] for p in build['protocols']) - or not any(s['name'] == 'pumas.http-advertisement' and s['version'] == 1 for s in build['schemas'])): - raise ValueError('Unsupported Pumas HTTP protocol/schema advertisement.') except (UnicodeError, RecursionError) as error: raise ValueError('Invalid Pumas descriptor encoding or nesting.') from error + if type(value) is not dict or 'advertisement_schema_version' not in value: + raise ValueError('Not a Pumas HTTP advertisement: advertisement_schema_version is missing.') + value = _object(value, '', ('advertisement_schema_version', 'service_generation', 'instance', 'endpoint', + 'build_info')) + _version(value['advertisement_schema_version'], 'advertisement_schema_version') + _text(value['service_generation'], 'service_generation', 128) + try: + endpoint(value['endpoint']) + except ValueError as error: + raise Rejected(f'Pumas advertisement endpoint is invalid: {error}') from None + instance = value['instance'] = _object( + value['instance'], 'instance', ('discovery_schema_version', 'registry_library_id', 'library_root', 'generation', + 'pumas_version', 'protocols', 'capabilities', 'model_ref_schema_version', + 'selector_schema_version'), ('build_info',)) + _version(instance['discovery_schema_version'], 'instance.discovery_schema_version') + for name in ('registry_library_id', 'library_root', 'generation', 'pumas_version'): + _text(instance[name], 'instance.' + name) + for name in ('model_ref_schema_version', 'selector_schema_version'): + _uint(instance[name], 'instance.' + name) + instance['protocols'] = _protocols(instance['protocols'], 'instance.protocols') + instance['capabilities'] = _strings(instance['capabilities'], 'instance.capabilities') + if instance.get('build_info') is not None: + instance['build_info'] = _build(instance['build_info'], 'instance.build_info') + build = value['build_info'] = _build(value['build_info'], 'build_info') + if not any(p['name'] == 'pumas.local-http' and 1 in p['versions'] for p in build['protocols']): + raise Rejected('Pumas advertisement build_info does not advertise protocol pumas.local-http version 1.') + if not any(s['name'] == 'pumas.http-advertisement' and s['version'] == 1 for s in build['schemas']): + raise Rejected('Pumas advertisement build_info does not advertise schema pumas.http-advertisement version 1.') return {'advertisement': value, 'observed_sha256': hashlib.sha256(raw).hexdigest()} diff --git a/pumas_operations.py b/pumas_operations.py index 89d345bc..3878473a 100644 --- a/pumas_operations.py +++ b/pumas_operations.py @@ -1,4 +1,9 @@ -"""Finite PR54 selected-model operations. Discovery budgets never time generation.""" +"""Finite Pumas typed model operations (contract version 1, v0.8 additive rules). + +Discovery budgets never time generation. Responses grow additively: unknown +response fields, capabilities, formats, option names, availability reasons and +error codes are ignored; only what Tuldok uses is validated and selected by name. +""" import hashlib import http.client import json @@ -13,20 +18,20 @@ from workbench import WorkbenchError, encode PROTOCOL = 'pumas_typed_v1' -SOURCE_COMMIT = '40c5cbfed67a6f0e862a1197bb5105363d67bdb1' -MAX_CAPABILITIES = 65536 +SOURCE_COMMIT = '4c79cd8e35db566b70caaa13d8d8f92265f5976e' +MAX_CAPABILITIES = 1024 * 1024 +MAX_SERVED_MODELS = 8 * 1024 * 1024 MAX_TEXT_RESPONSE = 256 * 1024 MAX_IMAGE_JSON = 12 * 1024 * 1024 MAX_DIMENSION = 2048 MAX_PIXELS = 4194304 -CAPABILITIES = {'chat_generation': 'chat_generation', 'text_generation': 'text_generation', - 'text_embedding': 'text_embedding', 'image_generation': 'text_to_image', - 'image_to_text': 'image_to_text', - 'audio_transcription': 'speech_to_text', 'audio_classification': 'audio_classification'} -ERROR_CODES = {'invalid_request', 'unsupported_contract', 'model_not_found', 'ambiguous_model', - 'ambiguous_operation', 'unsupported_modality', - 'capability_unavailable', 'provider_failure', 'invalid_provider_result', - 'request_limit', 'response_limit', 'transport_lost'} +MAX_MESSAGE = 600 +# Capabilities Tuldok sends, with their fixed contract-version-1 semantic task. +USED_CAPABILITIES = {'chat_generation': 'chat_generation', 'image_generation': 'text_to_image', + 'image_to_text': 'image_to_text'} +# Option bounds Tuldok compares against. Other advertised options are ignored. +USED_OPTIONS = {'max_tokens', 'temperature', 'top_p', 'width', 'height', 'seed', + 'image_bytes', 'image_pixels', 'image_count'} class OperationError(WorkbenchError): @@ -56,10 +61,63 @@ def constant(_): def object_fields(value, required, optional=()): + """Closed shape for request-side DTOs that Tuldok itself builds.""" if type(value) is not dict or not set(required) <= value.keys() or value.keys() - set(required) - set(optional): raise ValueError('Unsupported typed Pumas fields.') +def response_fields(value, required): + """Open shape for Pumas responses: required fields present, additive fields ignored.""" + if type(value) is not dict or not set(required) <= value.keys(): + raise ValueError('Typed Pumas response is missing required fields.') + + +def _positive_int(value): + return type(value) is int and value > 0 + + +def message_text(value): + """Bounded single-line display of Pumas' human-readable reason (never parsed).""" + if not isinstance(value, str): + return '' + value = ' '.join(''.join(c if c.isprintable() else ' ' for c in value).split()) + return value[:MAX_MESSAGE] + ('…' if len(value) > MAX_MESSAGE else '') + + +def typed_error(value): + """Decode `{contract_version, request_id, error:{code, outcome, message}}` or return None. + + `outcome` is authoritative. Only `not_admitted` proves no provider effect; + any other value is treated as `unknown`. Unknown codes are kept verbatim. + """ + if type(value) is not dict or type(value.get('error')) is not dict: + return None + error = value['error'] + code = error.get('code') + if not isinstance(code, str) or not code or len(code) > 200: + return None + outcome = 'not_admitted' if error.get('outcome') == 'not_admitted' else 'unknown' + return {'code': code, 'outcome': outcome, 'message': message_text(error.get('message'))} + + +def error_message(error, prefix='Pumas'): + detail = (': ' + error['message']) if error['message'] else '.' + if error['outcome'] == 'not_admitted': + return f"{prefix} refused the request ({error['code']}; not admitted, no provider effect){detail}" + return f"{prefix} reported {error['code']}; provider outcome unknown, no automatic replay{detail}" + + +def http_error(status, raw): + """Raise a readable ValueError for a non-200 typed discovery response.""" + try: + error = typed_error(strict_json(raw)) + except ValueError: + error = None + if error is None: + raise ValueError(f'Pumas returned HTTP {status} without a typed error body.') + raise ValueError(error_message(error)) + + def alias(value): if (not isinstance(value, str) or not value or value != value.strip() or len(value.encode('utf-8')) > 256 or any(ord(c) < 32 or ord(c) == 127 for c in value)): @@ -89,50 +147,69 @@ def configuration(body, *, image=False): return {'protocol': PROTOCOL, 'profile': profile(body['profile'])} +def _bound_number(value): + return type(value) in (int, float) and -1e308 <= value <= 1e308 + + +def _descriptor(item): + """Validate one descriptor Tuldok uses. Extra fields and enum values are ignored.""" + response_fields(item, ('capability', 'semantic_task', 'input_formats', 'output_formats', 'availability', 'option_bounds')) + name = item['capability'] + if item['semantic_task'] != USED_CAPABILITIES[name]: + raise ValueError('Pumas reports a different semantic task for '+name+' within contract version 1.') + for field in ('input_formats', 'output_formats'): + if type(item[field]) is not list or any(type(value) is not str for value in item[field]): + raise ValueError('Invalid typed Pumas '+field.replace('_', ' ')+'.') + availability = item['availability'] + if (type(availability) is not dict or type(availability.get('state')) is not str + or availability.get('reason') is not None and type(availability['reason']) is not str): + raise ValueError('Invalid typed Pumas availability.') + bounds = item['option_bounds'] + if type(bounds) is not list: + raise ValueError('Invalid typed Pumas option bounds.') + seen = set() + for bound in bounds: + if type(bound) is not dict or type(bound.get('option')) is not str or bound['option'] not in USED_OPTIONS: + continue + response_fields(bound, ('option', 'minimum', 'maximum')) + if bound['option'] in seen: + raise ValueError('Pumas advertises the '+bound['option']+' bound more than once.') + seen.add(bound['option']) + if not _bound_number(bound['minimum']) or not _bound_number(bound['maximum']) or bound['minimum'] > bound['maximum']: + raise ValueError('Invalid typed Pumas option interval.') + + def validate_manifest(raw): + """Validate the fields Tuldok uses from `GET /v1/capabilities`. + + Additive fields (for example `context_tokens`), unused capabilities such as + `multimodal_embedding` or `decision`, and unknown formats, reasons and option + names are accepted and ignored. The raw observation is retained unchanged. + """ value = strict_json(raw) - object_fields(value, ('supported_contract_versions', 'model', 'profile', 'max_request_bytes', - 'max_response_bytes', 'max_stream_event_bytes', 'capabilities')) + response_fields(value, ('supported_contract_versions', 'model', 'profile', 'max_request_bytes', + 'max_response_bytes', 'capabilities')) versions = value['supported_contract_versions'] - if type(versions) is not list or not 1 <= len(versions) <= 16 or any(type(v) is not int for v in versions) or 1 not in versions: + if type(versions) is not list or not any(type(version) is int and version == 1 for version in versions): raise ValueError('Pumas typed contract version 1 is unavailable.') alias(value['model']) if profile(value['profile']) is None: raise ValueError('Pumas must resolve an exact profile.') - for key, maximum in [('max_request_bytes', 32*1024*1024), ('max_response_bytes', 32*1024*1024), ('max_stream_event_bytes', 256*1024)]: - if type(value[key]) is not int or not 0 < value[key] <= maximum: + for key in ('max_request_bytes', 'max_response_bytes'): + if not _positive_int(value[key]): raise ValueError('Unsupported typed Pumas byte bounds.') entries = value['capabilities'] - if type(entries) is not list or len(entries) > len(CAPABILITIES): + if type(entries) is not list: raise ValueError('Invalid typed Pumas capability list.') seen = set() for item in entries: - object_fields(item, ('capability', 'semantic_task', 'input_formats', 'output_formats', 'streaming', 'availability', 'option_bounds')) - name = item['capability'] - if type(name) is not str or name not in CAPABILITIES or name in seen or item['semantic_task'] != CAPABILITIES[name] or type(item['streaming']) is not bool: - raise ValueError('Unsupported typed Pumas semantic capability.') + name = item.get('capability') if type(item) is dict else None + if type(name) is not str or name not in USED_CAPABILITIES: + continue + if name in seen: + raise ValueError('Pumas advertises '+name+' more than once for one serving.') seen.add(name) - for field, allowed in [('input_formats', {'messages_text', 'text', 'text_batch', 'pcm_s16le', 'pcm_f32le', 'png_base64', 'jpeg_base64', 'messages_image'}), - ('output_formats', {'text', 'embeddings_float32', 'png_base64', 'labels'})]: - items = item[field] - if type(items) is not list or not 1 <= len(items) <= len(allowed) or any(type(v) is not str or v not in allowed for v in items) or len(set(items)) != len(items): - raise ValueError('Unsupported typed Pumas format.') - availability = item['availability'] - object_fields(availability, ('state',), ('reason',)) - if availability not in [{'state': 'available'}] and (availability.get('state') != 'unavailable' or - type(availability.get('reason')) is not str or availability.get('reason') not in {'unsupported_adapter', 'unqualified_audio_runtime', 'unknown_model_task', 'model_task_mismatch', 'runtime_unavailable'}): - raise ValueError('Invalid typed Pumas availability.') - bounds = item['option_bounds'] - if type(bounds) is not list or len(bounds) > 13: - raise ValueError('Invalid typed Pumas option bounds.') - options = set() - for bound in bounds: - object_fields(bound, ('option', 'minimum', 'maximum')) - if type(bound['option']) is not str or bound['option'] not in {'max_tokens', 'temperature', 'top_p', 'dimensions', 'input_count', 'input_characters', 'width', 'height', 'seed', 'max_output_tokens', 'image_bytes', 'image_pixels', 'image_count'} or bound['option'] in options: - raise ValueError('Unsupported typed Pumas option.') - options.add(bound['option']) - if any(type(bound[k]) not in (int, float) or not -1e308 <= bound[k] <= 1e308 for k in ('minimum', 'maximum')) or bound['minimum'] > bound['maximum']: - raise ValueError('Invalid typed Pumas option interval.') + _descriptor(item) return {'capabilities': value, 'observed_sha256': hashlib.sha256(raw).hexdigest()} @@ -144,7 +221,8 @@ def capabilities(base, model, exact_profile=None): query['profile'] = exact_profile try: observed = gateway_discovery.get_json(parts.hostname, parts.port, '/v1/capabilities?' + urlencode(query), - time.monotonic()+3, max_response=MAX_CAPABILITIES, decode=validate_manifest, require_complete=True) + time.monotonic()+3, max_response=MAX_CAPABILITIES, decode=validate_manifest, require_complete=True, + error=http_error) except (ValueError, OSError, http.client.HTTPException) as error: raise OperationError('Selected-model capabilities unavailable: '+str(error)) from None manifest = observed['capabilities'] @@ -153,37 +231,102 @@ def capabilities(base, model, exact_profile=None): return observed +class _RouteMissing(ValueError): + pass + + +def _listing(aliases): + rows = [] + for name, profiles in aliases.items(): + required = len(profiles) > 1 + label = name + (' (profiles: ' + ', '.join(profiles) + '; choose one)' if required else '') + rows.append({'id': name, 'name': label, 'profiles': profiles, 'profile_required': required}) + return rows + + +def _served_models(raw): + """`GET /v1/served-models`: one entry per served (model, profile) pair.""" + value = strict_json(raw) + response_fields(value, ('models',)) + if type(value['models']) is not list: + raise ValueError('Invalid Pumas served-model listing.') + aliases = {} + for item in value['models']: + response_fields(item, ('model', 'profile')) + selected = profile(item['profile']) + if selected is None: + raise ValueError('Pumas served-model entry has no exact profile.') + profiles = aliases.setdefault(alias(item['model']), []) + if selected not in profiles: + profiles.append(selected) + return {'models': _listing(aliases), 'source': '/v1/served-models', + 'qualification': 'Served (model, profile) pairs. Inspect the selected model capabilities before admission.'} + + +def _openai_models(raw): + value = strict_json(raw) + entries = value.get('data') if type(value) is dict else None + if type(entries) is not list: + raise ValueError('Invalid bounded Pumas served model listing.') + aliases = {} + for item in entries: + if type(item) is not dict: + raise ValueError('Invalid model listing entry.') + profiles = aliases.setdefault(alias(item.get('id')), []) + # v0.8 lists each ID once with `profiles`; earlier gateways omit it. + listed = item.get('profiles') + for selected in listed if type(listed) is list else (): + if isinstance(selected, str) and re.fullmatch(r'[A-Za-z0-9_.-]{1,128}', selected) and selected not in profiles: + profiles.append(selected) + return {'models': _listing(aliases), 'source': '/v1/models', + 'qualification': 'Serving aliases only. Inspect the selected model capabilities before admission.'} + + def models(base): + """Prefer the typed served-model listing; fall back to `/v1/models` on gateways without it.""" parts = urlsplit(endpoint(base)) - def decode(raw): - value = strict_json(raw) - entries = value.get('data') if type(value) is dict else None - if type(entries) is not list or len(entries) > 512: - raise ValueError('Invalid bounded Pumas served model listing.') - names = [] - for item in entries: - if type(item) is not dict: - raise ValueError('Invalid model listing entry.') - name = alias(item.get('id')) - # PR54 emits one catalog row per served profile and omits profile - # in these rows. Exact-profile capabilities remain admission authority. - if name not in names: - names.append(name) - return {'models': [{'id': name, 'name': name} for name in names], - 'qualification': 'Serving aliases only. Inspect the selected model capabilities before admission.'} + def missing(status, raw): + if status == 404: + raise _RouteMissing() + http_error(status, raw) + try: + return gateway_discovery.get_json(parts.hostname, parts.port, '/v1/served-models', time.monotonic()+3, + max_response=MAX_SERVED_MODELS, decode=_served_models, require_complete=True, error=missing) + except _RouteMissing: + pass return gateway_discovery.get_json(parts.hostname, parts.port, '/v1/models', time.monotonic()+3, - max_response=1024*1024, decode=decode, require_complete=True) - - -def require(observed, name, input_format, output_format, options): - item = next((row for row in observed['capabilities']['capabilities'] if row['capability'] == name), None) - if (item is None or item['availability']['state'] != 'available' or input_format not in item['input_formats'] - or output_format not in item['output_formats']): - raise OperationError('Selected model does not currently advertise the required '+name+' capability.') - bounds = {row['option']: row for row in item['option_bounds']} + max_response=1024*1024, decode=_openai_models, require_complete=True, error=http_error) + + +def descriptor(observed, name): + """Select a capability descriptor by name, never by position or count.""" + return next((row for row in observed['capabilities']['capabilities'] + if type(row) is dict and row.get('capability') == name), None) + + +def require(observed, name, input_format, output_format, options, advisory=()): + """Check live availability and bounds. `advisory` options are checked only when advertised.""" + item = descriptor(observed, name) + if item is None: + raise OperationError('Selected model does not advertise the '+name+' capability.') + availability = item['availability'] + if availability.get('state') != 'available': + reason = availability.get('reason') + raise OperationError('Selected model reports '+name+' unavailable' + + (' ('+message_text(reason)+')' if isinstance(reason, str) else '') + '.') + if input_format not in item['input_formats'] or output_format not in item['output_formats']: + raise OperationError('Selected model does not advertise '+input_format+' to '+output_format+' for '+name+'.') + bounds = {row['option']: row for row in item['option_bounds'] + if type(row) is dict and type(row.get('option')) is str and row['option'] in USED_OPTIONS} for key, value in options.items(): - if key not in bounds or not bounds[key]['minimum'] <= value <= bounds[key]['maximum']: - raise OperationError('Selected model does not advertise the requested '+key+' bound.') + bound = bounds.get(key) + if bound is None: + if key in advisory: + continue + raise OperationError('Selected model does not advertise a '+key+' bound for '+name+'.') + if not bound['minimum'] <= value <= bound['maximum']: + raise OperationError(f"Requested {key} {value} is outside the selected model's {name} bound " + f"{bound['minimum']:g}–{bound['maximum']:g}.") def chat_request(job, user_text, max_tokens): @@ -236,9 +379,8 @@ def watch(): try: value = result(raw, payload['request_id']) except OperationError as error: + # The typed error's `outcome` is authoritative; HTTP status is evidence only. error.provider_http_status = response.status - if error.provider_code and response.status not in (400, 413, 422, 502, 503): - error.outcome = 'unknown' raise if value['kind'] != {'text': 'text', 'png_base64': 'image'}.get(payload['output']): raise OperationError('Typed result has the wrong output kind; no replay.', 'unknown', raw=raw) @@ -259,35 +401,45 @@ def watch(): def result(raw, request_id): + """Decode a finite typed response. Unknown response fields are ignored.""" try: value = strict_json(raw) - object_fields(value, ('contract_version', 'request_id'), ('result', 'error')) - if type(value['contract_version']) is not int or value['contract_version'] != 1 or value['request_id'] != request_id or ('result' in value) == ('error' in value): - raise ValueError('Wrong typed operation correlation.') + if type(value) is not dict or type(value.get('contract_version')) is not int or value['contract_version'] != 1: + raise ValueError('Typed response is not contract version 1.') if 'error' in value: - error = value['error']; object_fields(error, ('code', 'outcome')) - if type(error['code']) is not str or error['code'] not in ERROR_CODES or error['outcome'] not in ('not_admitted', 'unknown'): - raise ValueError('Invalid typed operation error.') - raise OperationError('Pumas '+error['code']+'; outcome '+error['outcome']+'. No automatic replay.', error['outcome'], error['code'], raw) + error = typed_error(value) + if error is None: + detail = message_text(value['error'].get('message')) if type(value['error']) is dict else '' + raise ValueError('Invalid typed operation error' + (': ' + detail if detail else '.')) + # A null request_id means Pumas could not read the body; it is still a typed rejection. + if value.get('request_id') not in (request_id, None): + raise ValueError('Typed error belongs to another request.') + raise OperationError(error_message(error), error['outcome'], error['code'], raw) + response_fields(value, ('request_id', 'result')) + if value['request_id'] != request_id: + raise ValueError('Typed result belongs to another request.') projected = value['result'] if type(projected) is not dict: raise ValueError('Invalid typed result.') if projected.get('kind') == 'text': - object_fields(projected, ('kind', 'text', 'finish_reason')) - if not isinstance(projected['text'], str) or projected['finish_reason'] != 'stop': - raise ValueError('A complete stop-terminated text result is required.') + response_fields(projected, ('kind', 'text', 'finish_reason')) + if not isinstance(projected['text'], str): + raise ValueError('Invalid typed text result.') + if projected['finish_reason'] != 'stop': + raise ValueError('Text did not finish with stop; partial output is not applied.') projected['text'].encode('utf-8') elif projected.get('kind') == 'image': - object_fields(projected, ('kind', 'png_base64', 'seed')) + response_fields(projected, ('kind', 'png_base64', 'seed')) if not isinstance(projected['png_base64'], str) or type(projected['seed']) is not int or not 0 <= projected['seed'] <= 4294967295: raise ValueError('Invalid typed image result.') else: - raise ValueError('Unsupported typed result format.') + raise ValueError('Unsupported typed result kind.') return projected except OperationError: raise - except (ValueError, UnicodeError, TypeError): - raise OperationError('Malformed or incomplete typed result; provider outcome unknown, no replay.', 'unknown', raw=raw) from None + except (ValueError, UnicodeError, TypeError) as error: + raise OperationError('Malformed or incomplete typed result; provider outcome unknown, no replay. '+str(error), + 'unknown', raw=raw) from None def image_dimensions(width, height): diff --git a/pumas_vision.py b/pumas_vision.py index 65ba2f2f..59a677ab 100644 --- a/pumas_vision.py +++ b/pumas_vision.py @@ -1,4 +1,4 @@ -"""Finite Image→Text consumer pinned to combined Pumas PR68, never model qualification. +"""Finite Image→Text consumer of the Pumas v0.8 typed contract, never model qualification. The application sends its prepared image bytes, without URL/path authority. Pumas's production PNG/turbojpeg decoder remains the admission authority. @@ -17,7 +17,7 @@ import pumas_operations as pumas from workbench import encode -SOURCE_COMMIT = '36c2aab2c885b8416fd4e39cf8c37c510596d6e4' +SOURCE_COMMIT = pumas.SOURCE_COMMIT SCHEMA = 'pumas.model-operations.image-to-text' MAX_REQUEST = 32 * 1024 * 1024 MAX_IMAGE = 8 * 1024 * 1024 @@ -26,6 +26,9 @@ MAX_AXIS = 4096 MAX_PIXELS = 4194304 MAX_MESSAGES = MAX_PARTS = 128 +# Pumas has no fixed generation cap; the live max_tokens bound follows the served context. +MAX_TOKENS = 4294967295 +IMAGE_FACTS = ('image_count', 'image_bytes', 'image_pixels', 'width', 'height') def _container(data, encoding): @@ -142,7 +145,8 @@ def validate_input(value, *, facade=False): def request(job, input_value, *, facade=True, max_tokens=512, temperature=None, top_p=None): - if type(max_tokens) is not int or not 1 <= max_tokens <= 2048: raise ValueError('Tokens must be 1–2048.') + if type(max_tokens) is not int or not 1 <= max_tokens <= MAX_TOKENS: + raise ValueError('Tokens must be a positive integer; the selected model bounds the maximum.') options = {'kind': 'text_generation', 'max_tokens': max_tokens} for key, value, maximum in [('temperature', temperature, 2), ('top_p', top_p, 1)]: if value is not None: @@ -171,7 +175,8 @@ def prepare(job, payload, stop): payload['profile'] = observed['capabilities']['profile'] input_format, images = validate_input(payload['input'], facade='semantic_task' in payload) options = {key: value for key, value in payload['options'].items() if key != 'kind'} - pumas.require(observed, 'image_to_text', input_format, 'text', dict(options, image_count=len(images))) - for image in images: pumas.require(observed, 'image_to_text', input_format, 'text', image) + # Image facts are checked against advertised bounds; local preflight already bounds them. + pumas.require(observed, 'image_to_text', input_format, 'text', dict(options, image_count=len(images)), advisory=IMAGE_FACTS) + for image in images: pumas.require(observed, 'image_to_text', input_format, 'text', image, advisory=IMAGE_FACTS) job.update(capability_observation=observed, build_observation=build, producer_contract_source=SOURCE_COMMIT) pumas.request_evidence(job, payload) diff --git a/static/pumas-gateways.js b/static/pumas-gateways.js index f6bc193c..6214e738 100644 --- a/static/pumas-gateways.js +++ b/static/pumas-gateways.js @@ -31,7 +31,9 @@ $('pumas-gateway-scan').addEventListener('click',async()=>{ pumasGateways=result.gateways; $('pumas-gateway-choice').replaceChildren(new Option('Choose an advertised gateway',''),...pumasGateways.map(row=> new Option(row.server_url+' · '+row.advertisement.build_info.package_version+' · '+row.advertisement.instance.registry_library_id,row.server_url))); - pumasGatewayStatus(pumasGateways.length?'Choose a gateway and annotation workflow. Model support is checked when you request a proposal.':'No compatible HTTP advertisements found. Start Pumas with HTTP discovery, or enter a legacy gateway URL in the proposal form.'); + // Advertisements Pumas served but Tuldok refused keep their reasons. + const refused=(Array.isArray(result.rejected)?result.rejected:[]).map(row=>' Refused '+row.server_url+': '+row.reason).join(''); + pumasGatewayStatus((pumasGateways.length?'Choose a gateway and annotation workflow. Model support is checked when you request a proposal.':'No compatible HTTP advertisements found. Start Pumas with HTTP discovery, or enter a legacy gateway URL in the proposal form.')+refused); }catch(error){if(token.epoch===pumasGatewayEpoch&&!pumasGatewayPaused)pumasGatewayStatus(error.message);} finally{if(pumasGatewayOperation===token){pumasGatewayOperation=null;renderPumasGateway();}} }); diff --git a/tests/browser_pumas_typed.cjs b/tests/browser_pumas_typed.cjs index ba87eab7..6dfa832f 100644 --- a/tests/browser_pumas_typed.cjs +++ b/tests/browser_pumas_typed.cjs @@ -117,7 +117,8 @@ function launch(command,args,options={}){const child=spawn(command,args,options) assert.equal(await evaluate('document.getElementById("caption-proposal-seed").disabled'),true); await click('caption-proposal-capabilities');await until(()=>evaluate('document.getElementById("caption-proposal-capability-metadata").textContent.includes("image_to_text")')); const visionManifest=JSON.parse(await evaluate('document.getElementById("caption-proposal-capability-metadata").textContent')); - assert.equal(visionManifest.capabilities.capabilities.length,7); + // Pumas responses grow additively; require the capability this flow uses, not a fixed count. + assert.ok(visionManifest.capabilities.capabilities.some(d=>d.capability==='image_to_text')); await evaluate('document.getElementById("caption-proposal-form").requestSubmit()'); const visionJob=await until(async()=> (await request('caption-proposals')).jobs.find(j=>j.source.id===visionSource.id && j.status==='completed')); const visionExact=await request('caption-proposals/'+visionJob.id); diff --git a/tests/fake_ai.py b/tests/fake_ai.py index 75e4fb2d..d74e7f34 100644 --- a/tests/fake_ai.py +++ b/tests/fake_ai.py @@ -26,6 +26,12 @@ def do_POST(self): self.send_response(404);self.send_header('Content-Length','0');self.end_headers();return body=json.loads(self.rfile.read(int(self.headers['Content-Length']))) requests.append((self.path,body,dict(self.headers))) + if body['model']=='pumas-conflict': + encoded=json.dumps({'error':{'message':'model corners-test is served by profiles a, b; select one','type':'pumas_error','code':'ambiguous_model'}}).encode() + self.send_response(409);self.send_header('Content-Length',str(len(encoded)));self.end_headers();self.wfile.write(encoded);return + if body['model']=='plain-conflict': + encoded=json.dumps({'error':{'message':'private upstream wording'}}).encode() + self.send_response(409);self.send_header('Content-Length',str(len(encoded)));self.end_headers();self.wfile.write(encoded);return if body['model']=='missing-projector': encoded=json.dumps({'error':{'message':'image input is not supported - provide the mmproj'}}).encode() self.send_response(500);self.send_header('Content-Length',str(len(encoded)));self.end_headers();self.wfile.write(encoded);return diff --git a/tests/fake_images.py b/tests/fake_images.py index 885af67c..f7103fab 100644 --- a/tests/fake_images.py +++ b/tests/fake_images.py @@ -50,6 +50,9 @@ def do_POST(self): return if body['prompt'] == 'busy': return self.reply({'error': {'code': 'runtime_busy'}}, 409) + if body['prompt'] == 'pumas-refusal': + return self.reply({'error': {'message': 'width 33 is not a multiple of 8 for this model; token=sk-abcdefghijkl', + 'type': 'pumas_error', 'code': 'invalid_request'}}, 400) if body['prompt'] == 'unprocessable': return self.reply({'detail': 'fixture contract mismatch'}, 422) raw = b'invalid PNG' if body['prompt'] == 'invalid' else png((body['width'], body['height']), '#467e9a' if body.get('seed', 7) == 7 else '#%06x' % (body['seed'] % 16777216)) diff --git a/tests/fake_pumas_typed.py b/tests/fake_pumas_typed.py index b402ba68..e28f311e 100644 --- a/tests/fake_pumas_typed.py +++ b/tests/fake_pumas_typed.py @@ -24,6 +24,8 @@ def reply(self, value, status=200): try: self.wfile.write(raw) except OSError: pass def do_GET(self): + if self.path == '/v1/served-models' and state.get('served_models') is not None: + return self.reply(state['served_models']) if self.path == '/v1/models': rows = [{'id': 'controlled-text'}, {'id': 'controlled-image'}] if state.get('duplicate_alias'): rows.append({'id': 'controlled-text'}) @@ -31,7 +33,13 @@ def do_GET(self): query = parse_qs(urlsplit(self.path).query) model = query.get('model', [''])[0] if model not in ('controlled-text', 'controlled-image'): + if urlsplit(self.path).path == '/v1/capabilities': + return self.reply({'contract_version': 1, 'request_id': None, 'error': {'code': 'model_not_found', + 'outcome': 'not_admitted', 'message': 'no served model is named ' + repr(model)}}, 404) return self.reply({'error': 'unknown model'}, 404) + if state.get('ambiguous'): + return self.reply({'contract_version': 1, 'request_id': None, 'error': {'code': 'ambiguous_model', + 'outcome': 'not_admitted', 'message': 'alias is served by profiles a, b; send profile'}}, 409) manifest = json.loads((FIXTURES / (model+'-capabilities.json')).read_bytes()) if state['mode'] == 'wrong_profile': manifest['profile'] = 'foreign' if state['mode'] == 'unavailable': @@ -47,7 +55,11 @@ def do_POST(self): if mode == 'loss': self.close_connection = True; return if mode in ('not_admitted', 'unknown'): return self.reply({'contract_version': 1, 'request_id': body['request_id'], - 'error': {'code': 'capability_unavailable' if mode == 'not_admitted' else 'provider_failure', 'outcome': mode}}, 503) + 'error': {'code': 'capability_unavailable' if mode == 'not_admitted' else 'provider_failure', 'outcome': mode, + 'message': state.get('error_message', 'fixture provider reason')}}, state.get('error_status', 503)) + if mode == 'unreadable': + return self.reply({'contract_version': 1, 'request_id': None, 'error': {'code': 'invalid_request', + 'outcome': 'not_admitted', 'message': 'request body is not valid JSON'}}, 400) result = {'kind': 'text', 'text': state['content'], 'finish_reason': 'stop'} if body['capability'] == 'image_generation': import io diff --git a/tests/fake_pumas_vision.py b/tests/fake_pumas_vision.py index 2da5684c..a1eda85e 100644 --- a/tests/fake_pumas_vision.py +++ b/tests/fake_pumas_vision.py @@ -35,7 +35,16 @@ def do_GET(self): value = json.loads((FIXTURES / 'controlled-capabilities.json').read_bytes()) query = parse_qs(urlsplit(self.path).query) if query.get('profile', ['vision-cpu'])[0] != 'vision-cpu': return self.reply({},404) - if state['mode'] == 'unavailable': value['capabilities'][-1]['availability'] = {'state':'unavailable','reason':'runtime_unavailable'} + vision = next(row for row in value['capabilities'] if row['capability'] == 'image_to_text') + if state['mode'] == 'unavailable': vision['availability'] = {'state':'unavailable','reason':'runtime_unavailable'} + if state['mode'] == 'additive': + # v0.8 additive growth: new fields, capabilities, formats, options and reasons. + value['future_field'] = {'nested': True} + value['capabilities'].insert(0, {'capability':'future_capability','semantic_task':'future_task','input_formats':['future'], + 'output_formats':['future'],'streaming':False,'availability':{'state':'deferred'},'option_bounds':[],'extra':1}) + vision['input_formats'].append('future_format'); vision['future'] = 'ignored' + vision['option_bounds'].insert(0, {'option':'future_option','minimum':0.0,'maximum':1.0,'step':1}) + value['capabilities'][-1]['availability'] = {'state':'unavailable','reason':'future_reason'} if state['mode'] == 'wrong_profile': value['profile'] = 'foreign' if state['mode'] == 'wrong_model': value['model'] = 'foreign' return self.reply(value) @@ -47,11 +56,15 @@ def do_POST(self): if mode == 'loss': self.close_connection = True; return if mode in ('not_admitted','unknown'): return self.reply({'contract_version':1,'request_id':body['request_id'], - 'error':{'code':'unsupported_modality' if mode=='not_admitted' else 'invalid_provider_result','outcome':mode}},state['refusal_status']) + 'error':{'code':state.get('error_code') or ('unsupported_modality' if mode=='not_admitted' else 'invalid_provider_result'), + 'outcome':mode,'message':state.get('error_message','fixture reason')}},state['refusal_status']) + if mode == 'unreadable': + return self.reply({'contract_version':1,'request_id':None, + 'error':{'code':'invalid_request','outcome':'not_admitted','message':'request body is not valid JSON'}},400) value = {'contract_version':1,'request_id':body['request_id'], 'result':{'kind':'text','text':state['content'],'finish_reason':'stop'}} if mode == 'wrong_id': value['request_id'] = 'foreign' - if mode == 'private': value['model'] = 'provider/private' + if mode == 'additive': value['future'] = {'x': 1}; value['result']['future'] = 'ignored' if mode == 'length': value['result']['finish_reason'] = 'length' if mode == 'nonfinite': value['extra'] = float('nan') if mode == 'oversized': value['result']['text'] = 'x'*300000 diff --git a/tests/fixtures/pumas-image-to-text-v1/controlled-capabilities.json b/tests/fixtures/pumas-image-to-text-v1/controlled-capabilities.json index 27a4122a..262544db 100644 --- a/tests/fixtures/pumas-image-to-text-v1/controlled-capabilities.json +++ b/tests/fixtures/pumas-image-to-text-v1/controlled-capabilities.json @@ -7,6 +7,7 @@ "max_request_bytes": 33554432, "max_response_bytes": 33554432, "max_stream_event_bytes": 262144, + "context_tokens": 8192, "capabilities": [ { "capability": "chat_generation", @@ -25,7 +26,7 @@ { "option": "max_tokens", "minimum": 1.0, - "maximum": 4294967295.0 + "maximum": 8192.0 }, { "option": "temperature", @@ -56,7 +57,7 @@ { "option": "max_tokens", "minimum": 1.0, - "maximum": 4294967295.0 + "maximum": 8192.0 }, { "option": "temperature", @@ -89,17 +90,7 @@ { "option": "dimensions", "minimum": 1.0, - "maximum": 8192.0 - }, - { - "option": "input_count", - "minimum": 1.0, - "maximum": 128.0 - }, - { - "option": "input_characters", - "minimum": 0.0, - "maximum": 65536.0 + "maximum": 4294967295.0 } ] }, @@ -158,8 +149,8 @@ "option_bounds": [ { "option": "max_output_tokens", - "minimum": 512, - "maximum": 512 + "minimum": 512.0, + "maximum": 512.0 } ] }, @@ -176,7 +167,7 @@ "streaming": false, "availability": { "state": "unavailable", - "reason": "unqualified_audio_runtime" + "reason": "unsupported_adapter" }, "option_bounds": [] }, @@ -198,43 +189,114 @@ "option_bounds": [ { "option": "max_tokens", - "minimum": 1, - "maximum": 2048 + "minimum": 1.0, + "maximum": 8192.0 }, { "option": "temperature", - "minimum": 0, - "maximum": 2 + "minimum": 0.0, + "maximum": 2.0 }, { "option": "top_p", - "minimum": 0, - "maximum": 1 + "minimum": 0.0, + "maximum": 1.0 }, { "option": "image_bytes", - "minimum": 1, - "maximum": 8388608 + "minimum": 1.0, + "maximum": 8388608.0 }, { "option": "image_pixels", - "minimum": 1, - "maximum": 4194304 + "minimum": 1.0, + "maximum": 4194304.0 }, { "option": "image_count", - "minimum": 1, - "maximum": 4 + "minimum": 1.0, + "maximum": 4.0 }, { "option": "width", - "minimum": 1, - "maximum": 4096 + "minimum": 1.0, + "maximum": 4096.0 }, { "option": "height", - "minimum": 1, - "maximum": 4096 + "minimum": 1.0, + "maximum": 4096.0 + } + ] + }, + { + "capability": "multimodal_embedding", + "semantic_task": "multimodal_embedding", + "input_formats": [ + "png_base64", + "jpeg_base64", + "content_batch" + ], + "output_formats": [ + "embeddings_float32" + ], + "streaming": false, + "availability": { + "state": "unavailable", + "reason": "model_task_mismatch" + }, + "option_bounds": [ + { + "option": "dimensions", + "minimum": 1.0, + "maximum": 4294967295.0 + }, + { + "option": "image_bytes", + "minimum": 1.0, + "maximum": 8388608.0 + }, + { + "option": "image_pixels", + "minimum": 1.0, + "maximum": 4194304.0 + }, + { + "option": "image_count", + "minimum": 1.0, + "maximum": 4.0 + } + ] + }, + { + "capability": "decision", + "semantic_task": "decision", + "input_formats": [ + "decision" + ], + "output_formats": [ + "decisions" + ], + "streaming": false, + "availability": { + "state": "unavailable", + "reason": "model_task_mismatch" + }, + "option_bounds": [ + { + "option": "image_bytes", + "minimum": 1.0, + "maximum": 8388608.0 + }, + { + "option": "image_pixels", + "minimum": 1.0, + "maximum": 4194304.0 + }, + { + "option": "image_count", + "minimum": 0.0, + "maximum": 4.0 } ] } diff --git a/tests/fixtures/pumas-image-to-text-v1/schemas/capabilities-response.schema.json b/tests/fixtures/pumas-image-to-text-v1/schemas/capabilities-response.schema.json index d9534ecd..c455d4f7 100644 --- a/tests/fixtures/pumas-image-to-text-v1/schemas/capabilities-response.schema.json +++ b/tests/fixtures/pumas-image-to-text-v1/schemas/capabilities-response.schema.json @@ -43,16 +43,30 @@ "type": "string" }, "Capability": { - "enum": [ - "chat_generation", - "text_generation", - "text_embedding", - "image_generation", - "image_to_text", - "audio_transcription", - "audio_classification" - ], - "type": "string" + "oneOf": [ + { + "enum": [ + "chat_generation", + "text_generation", + "text_embedding", + "image_generation", + "image_to_text", + "audio_transcription", + "audio_classification" + ], + "type": "string" + }, + { + "const": "multimodal_embedding", + "description": "One embedding per image or per mixed text/image item, in the same\nvector space as the model's text embeddings.", + "type": "string" + }, + { + "const": "decision", + "description": "Typed questions about a state answered with probabilities by a native\ndecision model (llama.cpp `/v1/systemone`, e.g. Kev, OpenJev).", + "type": "string" + } + ] }, "CapabilityDescriptor": { "properties": { @@ -107,7 +121,9 @@ "text", "text_batch", "pcm_s16le", - "pcm_f32le" + "pcm_f32le", + "content_batch", + "decision" ], "type": "string" }, @@ -155,7 +171,8 @@ "text", "embeddings_float32", "png_base64", - "labels" + "labels", + "decisions" ], "type": "string" }, @@ -167,7 +184,9 @@ "text_to_image", "speech_to_text", "audio_classification", - "image_to_text" + "image_to_text", + "multimodal_embedding", + "decision" ], "type": "string" } @@ -180,6 +199,15 @@ }, "type": "array" }, + "context_tokens": { + "description": "Context window configured for this serving, when the runtime reports it.", + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, "max_request_bytes": { "format": "uint", "minimum": 0, diff --git a/tests/fixtures/pumas-image-to-text-v1/schemas/capability-descriptor.schema.json b/tests/fixtures/pumas-image-to-text-v1/schemas/capability-descriptor.schema.json index 0377d31b..9dc9d126 100644 --- a/tests/fixtures/pumas-image-to-text-v1/schemas/capability-descriptor.schema.json +++ b/tests/fixtures/pumas-image-to-text-v1/schemas/capability-descriptor.schema.json @@ -43,16 +43,30 @@ "type": "string" }, "Capability": { - "enum": [ - "chat_generation", - "text_generation", - "text_embedding", - "image_generation", - "image_to_text", - "audio_transcription", - "audio_classification" - ], - "type": "string" + "oneOf": [ + { + "enum": [ + "chat_generation", + "text_generation", + "text_embedding", + "image_generation", + "image_to_text", + "audio_transcription", + "audio_classification" + ], + "type": "string" + }, + { + "const": "multimodal_embedding", + "description": "One embedding per image or per mixed text/image item, in the same\nvector space as the model's text embeddings.", + "type": "string" + }, + { + "const": "decision", + "description": "Typed questions about a state answered with probabilities by a native\ndecision model (llama.cpp `/v1/systemone`, e.g. Kev, OpenJev).", + "type": "string" + } + ] }, "InputFormat": { "enum": [ @@ -63,7 +77,9 @@ "text", "text_batch", "pcm_s16le", - "pcm_f32le" + "pcm_f32le", + "content_batch", + "decision" ], "type": "string" }, @@ -111,7 +127,8 @@ "text", "embeddings_float32", "png_base64", - "labels" + "labels", + "decisions" ], "type": "string" }, @@ -123,7 +140,9 @@ "text_to_image", "speech_to_text", "audio_classification", - "image_to_text" + "image_to_text", + "multimodal_embedding", + "decision" ], "type": "string" } diff --git a/tests/fixtures/pumas-image-to-text-v1/schemas/error-response.schema.json b/tests/fixtures/pumas-image-to-text-v1/schemas/error-response.schema.json index 89f00acb..a7694c93 100644 --- a/tests/fixtures/pumas-image-to-text-v1/schemas/error-response.schema.json +++ b/tests/fixtures/pumas-image-to-text-v1/schemas/error-response.schema.json @@ -22,13 +22,18 @@ "code": { "$ref": "#/$defs/ErrorCode" }, + "message": { + "description": "Human-readable reason. Consumers display or log it; they branch on `code`.", + "type": "string" + }, "outcome": { "$ref": "#/$defs/Outcome" } }, "required": [ "code", - "outcome" + "outcome", + "message" ], "type": "object" }, diff --git a/tests/fixtures/pumas-image-to-text-v1/schemas/modality-request.schema.json b/tests/fixtures/pumas-image-to-text-v1/schemas/modality-request.schema.json index dc979861..0f014046 100644 --- a/tests/fixtures/pumas-image-to-text-v1/schemas/modality-request.schema.json +++ b/tests/fixtures/pumas-image-to-text-v1/schemas/modality-request.schema.json @@ -27,6 +27,32 @@ ], "type": "string" }, + "Capability": { + "oneOf": [ + { + "enum": [ + "chat_generation", + "text_generation", + "text_embedding", + "image_generation", + "image_to_text", + "audio_transcription", + "audio_classification" + ], + "type": "string" + }, + { + "const": "multimodal_embedding", + "description": "One embedding per image or per mixed text/image item, in the same\nvector space as the model's text embeddings.", + "type": "string" + }, + { + "const": "decision", + "description": "Typed questions about a state answered with probabilities by a native\ndecision model (llama.cpp `/v1/systemone`, e.g. Kev, OpenJev).", + "type": "string" + } + ] + }, "Content": { "anyOf": [ { @@ -40,6 +66,71 @@ } ] }, + "ContentItem": { + "additionalProperties": false, + "properties": { + "parts": { + "items": { + "$ref": "#/$defs/ImagePart" + }, + "type": "array" + } + }, + "required": [ + "parts" + ], + "type": "object" + }, + "DecisionImage": { + "additionalProperties": false, + "properties": { + "data_base64": { + "type": "string" + }, + "encoding": { + "$ref": "#/$defs/ImageEncoding" + } + }, + "required": [ + "encoding", + "data_base64" + ], + "type": "object" + }, + "DecisionQuestion": { + "additionalProperties": false, + "properties": { + "criteria": true, + "instructions": true, + "type": { + "$ref": "#/$defs/DecisionType" + } + }, + "required": [ + "type", + "instructions" + ], + "type": "object" + }, + "DecisionType": { + "oneOf": [ + { + "const": "choice", + "description": "Pick one option; `criteria` maps option to description (or null).", + "type": "string" + }, + { + "const": "score", + "description": "Rate on a scale; `criteria` lists level descriptions, lowest first.", + "type": "string" + }, + { + "const": "noul", + "description": "Probability that the answer is true; optional `criteria` describes\n`true` and `false`.", + "type": "string" + } + ] + }, "ImageEncoding": { "enum": [ "png", @@ -47,6 +138,67 @@ ], "type": "string" }, + "ImageMessage": { + "additionalProperties": false, + "properties": { + "content": { + "items": { + "$ref": "#/$defs/ImagePart" + }, + "type": "array" + }, + "role": { + "$ref": "#/$defs/Role" + } + }, + "required": [ + "role", + "content" + ], + "type": "object" + }, + "ImagePart": { + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "text", + "type": "string" + }, + "text": { + "type": "string" + } + }, + "required": [ + "kind", + "text" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "data_base64": { + "type": "string" + }, + "encoding": { + "$ref": "#/$defs/ImageEncoding" + }, + "kind": { + "const": "image", + "type": "string" + } + }, + "required": [ + "kind", + "encoding", + "data_base64" + ], + "type": "object" + } + ] + }, "Input": { "oneOf": [ { @@ -106,6 +258,27 @@ ], "type": "object" }, + { + "additionalProperties": false, + "description": "Messages whose content is a list of text and image parts.", + "properties": { + "kind": { + "const": "image_messages", + "type": "string" + }, + "messages": { + "items": { + "$ref": "#/$defs/ImageMessage" + }, + "type": "array" + } + }, + "required": [ + "kind", + "messages" + ], + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -166,6 +339,57 @@ "data_base64" ], "type": "object" + }, + { + "additionalProperties": false, + "description": "Items to embed together; each item produces one vector.", + "properties": { + "items": { + "items": { + "$ref": "#/$defs/ContentItem" + }, + "type": "array" + }, + "kind": { + "const": "content_batch", + "type": "string" + } + }, + "required": [ + "kind", + "items" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "A state and typed questions for a native decision model.", + "properties": { + "images": { + "default": [], + "items": { + "$ref": "#/$defs/DecisionImage" + }, + "type": "array" + }, + "kind": { + "const": "decision", + "type": "string" + }, + "questions": { + "additionalProperties": { + "$ref": "#/$defs/DecisionQuestion" + }, + "type": "object" + }, + "state": true + }, + "required": [ + "kind", + "state", + "questions" + ], + "type": "object" } ] }, @@ -313,6 +537,19 @@ "kind" ], "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "decision", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" } ] }, @@ -323,7 +560,8 @@ "png_base64", "labels", "pcm_s16le", - "pcm_f32le" + "pcm_f32le", + "decisions" ], "type": "string" }, @@ -425,7 +663,9 @@ "text_to_image", "speech_to_text", "audio_classification", - "image_to_text" + "image_to_text", + "multimodal_embedding", + "decision" ], "type": "string" } @@ -433,6 +673,18 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, "properties": { + "capability": { + "anyOf": [ + { + "$ref": "#/$defs/Capability" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional explicit capability. When present, resolution only considers it." + }, "contract_version": { "format": "uint32", "minimum": 0, diff --git a/tests/fixtures/pumas-image-to-text-v1/schemas/operation-request.schema.json b/tests/fixtures/pumas-image-to-text-v1/schemas/operation-request.schema.json index ee426cd0..84219b50 100644 --- a/tests/fixtures/pumas-image-to-text-v1/schemas/operation-request.schema.json +++ b/tests/fixtures/pumas-image-to-text-v1/schemas/operation-request.schema.json @@ -28,16 +28,95 @@ "type": "string" }, "Capability": { - "enum": [ - "chat_generation", - "text_generation", - "text_embedding", - "image_generation", - "image_to_text", - "audio_transcription", - "audio_classification" + "oneOf": [ + { + "enum": [ + "chat_generation", + "text_generation", + "text_embedding", + "image_generation", + "image_to_text", + "audio_transcription", + "audio_classification" + ], + "type": "string" + }, + { + "const": "multimodal_embedding", + "description": "One embedding per image or per mixed text/image item, in the same\nvector space as the model's text embeddings.", + "type": "string" + }, + { + "const": "decision", + "description": "Typed questions about a state answered with probabilities by a native\ndecision model (llama.cpp `/v1/systemone`, e.g. Kev, OpenJev).", + "type": "string" + } + ] + }, + "ContentItem": { + "additionalProperties": false, + "properties": { + "parts": { + "items": { + "$ref": "#/$defs/ImagePart" + }, + "type": "array" + } + }, + "required": [ + "parts" ], - "type": "string" + "type": "object" + }, + "DecisionImage": { + "additionalProperties": false, + "properties": { + "data_base64": { + "type": "string" + }, + "encoding": { + "$ref": "#/$defs/ImageEncoding" + } + }, + "required": [ + "encoding", + "data_base64" + ], + "type": "object" + }, + "DecisionQuestion": { + "additionalProperties": false, + "properties": { + "criteria": true, + "instructions": true, + "type": { + "$ref": "#/$defs/DecisionType" + } + }, + "required": [ + "type", + "instructions" + ], + "type": "object" + }, + "DecisionType": { + "oneOf": [ + { + "const": "choice", + "description": "Pick one option; `criteria` maps option to description (or null).", + "type": "string" + }, + { + "const": "score", + "description": "Rate on a scale; `criteria` lists level descriptions, lowest first.", + "type": "string" + }, + { + "const": "noul", + "description": "Probability that the answer is true; optional `criteria` describes\n`true` and `false`.", + "type": "string" + } + ] }, "ImageEncoding": { "enum": [ @@ -262,6 +341,60 @@ "data_base64" ], "type": "object" + }, + { + "additionalProperties": false, + "description": "Items to embed; each item's text and image parts produce one vector.", + "properties": { + "items": { + "items": { + "$ref": "#/$defs/ContentItem" + }, + "type": "array" + }, + "kind": { + "const": "content_batch", + "type": "string" + } + }, + "required": [ + "kind", + "items" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "A state and typed questions for a decision model.", + "properties": { + "images": { + "description": "Optional images, for decision models with a loaded projector.", + "items": { + "$ref": "#/$defs/DecisionImage" + }, + "type": "array" + }, + "kind": { + "const": "decision", + "type": "string" + }, + "questions": { + "additionalProperties": { + "$ref": "#/$defs/DecisionQuestion" + }, + "description": "Question id to question. Each question is answered independently.", + "type": "object" + }, + "state": { + "description": "Content to evaluate: a string, or JSON given to the model as text." + } + }, + "required": [ + "kind", + "state", + "questions" + ], + "type": "object" } ] }, @@ -393,6 +526,19 @@ "kind" ], "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "decision", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" } ] }, @@ -401,7 +547,8 @@ "text", "embeddings_float32", "png_base64", - "labels" + "labels", + "decisions" ], "type": "string" }, diff --git a/tests/fixtures/pumas-image-to-text-v1/schemas/operation-response.schema.json b/tests/fixtures/pumas-image-to-text-v1/schemas/operation-response.schema.json index 82857c00..277d7a17 100644 --- a/tests/fixtures/pumas-image-to-text-v1/schemas/operation-response.schema.json +++ b/tests/fixtures/pumas-image-to-text-v1/schemas/operation-response.schema.json @@ -1,5 +1,94 @@ { "$defs": { + "DecisionAnswer": { + "oneOf": [ + { + "properties": { + "choice": { + "type": "string" + }, + "confidence": { + "format": "double", + "type": "number" + }, + "probabilities": { + "additionalProperties": { + "format": "double", + "type": "number" + }, + "type": "object" + }, + "type": { + "const": "choice", + "type": "string" + } + }, + "required": [ + "type", + "choice", + "probabilities", + "confidence" + ], + "type": "object" + }, + { + "properties": { + "confidence": { + "format": "double", + "type": "number" + }, + "legend": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "probabilities": { + "additionalProperties": { + "format": "double", + "type": "number" + }, + "type": "object" + }, + "score": { + "description": "Probability-weighted level index; may fall between levels.", + "format": "double", + "type": "number" + }, + "type": { + "const": "score", + "type": "string" + } + }, + "required": [ + "type", + "score", + "legend", + "probabilities", + "confidence" + ], + "type": "object" + }, + { + "properties": { + "noul": { + "description": "Probability that the answer is true.", + "format": "double", + "type": "number" + }, + "type": { + "const": "noul", + "type": "string" + } + }, + "required": [ + "type", + "noul" + ], + "type": "object" + } + ] + }, "FinishReason": { "enum": [ "stop", @@ -74,6 +163,34 @@ "seed" ], "type": "object" + }, + { + "properties": { + "answers": { + "additionalProperties": { + "$ref": "#/$defs/DecisionAnswer" + }, + "type": "object" + }, + "input_tokens": { + "description": "Prompt tokens evaluated across all questions, when reported.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "kind": { + "const": "decisions", + "type": "string" + } + }, + "required": [ + "kind", + "answers" + ], + "type": "object" } ] } diff --git a/tests/fixtures/pumas-image-to-text-v1/source.json b/tests/fixtures/pumas-image-to-text-v1/source.json index e3254144..40d6dc5d 100644 --- a/tests/fixtures/pumas-image-to-text-v1/source.json +++ b/tests/fixtures/pumas-image-to-text-v1/source.json @@ -1,32 +1,45 @@ { "producer_repository": "MrScripty/Pumas-Library", - "producer_pr": 68, - "producer_commit": "36c2aab2c885b8416fd4e39cf8c37c510596d6e4", - "contract_url": "https://github.com/MrScripty/Pumas-Library/blob/36c2aab2c885b8416fd4e39cf8c37c510596d6e4/docs/contracts/image-to-text-v0.8.md", + "producer_branch": "feat/v080-consumer-api", + "producer_commit": "4c79cd8e35db566b70caaa13d8d8f92265f5976e", + "producer_tree": "c646a01d5cc88814bdd2dd7e4565d9d1deda2203", + "contract_path": "docs/contracts/model-operations-v1.md", + "contract_url": "https://github.com/MrScripty/Pumas-Library/blob/4c79cd8e35db566b70caaa13d8d8f92265f5976e/docs/contracts/model-operations-v1.md", + "contract_sha256": "ab0f8a2c9b01f89a28d9a03ed3a9d9246eb6d1665ee2690a4fd2bcb8ff0f9d44", "fixture_kind": "Source-derived controlled consumer fixture; not actual native DTO export or model inference", "inspected_source_sha256": { - "types.rs": "875977948163c4256e60783d540337f117ec70edb1bb3636e7598e516587cca3", - "model_operations.rs": "7c4dfac738db1af29b576691ee929f873b141c7c8e9549e11bc44c4ee1e05c28", - "projection.rs": "444c500edd7ca3ea478d37e68212f8da9325028b87a35f6cdedb3e0e8aaed7a7", - "vision_tests.rs": "e2c175c8d1eab53227d1a91b0fca4621838081ec1d587325ff29312f93be41d6" + "types.rs": "dc4e11a3de059f268c7bbf0f2008b4d1710f5ca6e8c20d06608e67773579bbd0", + "model_operations.rs": "2ffd1b7e215fc0da59fbb70acfbd6a5beb2aa4370fb15546afc7a16f6b49ef16", + "projection.rs": "7357166a8ef9e73b29602323b6ef3d7cca3364b9a18ac0a4715f467e9080f9d8", + "vision_tests.rs": "ce340d0dfe576196330e85b6a92151e6d7329dd9878a0c39e52f0f8126a0dc5a" }, - "schema_export_status": "Six actual production Rust DTO schemas were regenerated at schema_export_commit. The reviewed producer retains identical DTO definitions and schema dependencies; all six SHA-256 identities are unchanged. Handler and controlled-test source hashes are refreshed. The merged main source tree matches the reviewed producer exactly. Packaged/native qualification remains separate.", "inspected_source_native_blob_sha": { - "types.rs": "f7d70f90770deb124dc17dde5212d88b058d9e0e", - "model_operations.rs": "13d15ea4805cfc4df103951d974d76f8987d2964", - "projection.rs": "2a2061a0659a916998455bd22cbc55a7b7f4b0a6", - "vision_tests.rs": "531ff6b48c450800e860a5bc98ea7f00fe27fd3b" + "types.rs": "4b5414db04e62af6d1e39a3c39455ed6bc23e546", + "model_operations.rs": "64ed36cf3606f3028a3a2ec641d02790e71e56fc", + "projection.rs": "c1595de051af74c752e2553e6d2f3c5159c1f47e", + "vision_tests.rs": "4de87a94aba30ec82f19e49c4a8e47ef6de49654" }, - "originating_vision_pr": 65, + "schema_export_status": "Six actual production Rust DTO schemas were regenerated at schema_export_commit with the export-contract feature. The controlled capability fixture follows that commit's descriptor construction for a llama.cpp vision serving with an 8192-token context. Packaged/native qualification remains separate.", "actual_dto_schema_sha256": { - "capabilities-response.schema.json": "b55b150f6958239e71d71f1b28fd36450907ddccf78544d52b64543ad4f2493d", - "capability-descriptor.schema.json": "eb25783123d38de5ff6aedf4cfa69de8f4f5f5c60799768b1b90da258b85f922", - "error-response.schema.json": "b024363b19d55a004d9577ad04d7e2ec9209d3d1193b7c53c46a7d6ed401e3be", - "modality-request.schema.json": "24b5f0a73a1252d31a3b22687a84f07b16a9d3ea700809a42e615e7f4e668704", - "operation-request.schema.json": "08f46f0e79c6cb288c413b008abbdfbedb1898d02fb98d396ef73faf8634c642", - "operation-response.schema.json": "20bf98dfd7292b1e3a99bf34ec524858351ba6f9d226f17257362e9ddf0500f8" + "capabilities-response.schema.json": "e1949bef5e7480f2808f2f1ec66e77adf438e44c6d000a04430bbf15d117a04e", + "capability-descriptor.schema.json": "c586a54fd8c0524dfcbae00dd298d5ee06bb70675c78e871b8a82a77c7ba5316", + "error-response.schema.json": "6b681ab7c7f224e2a35dddaeeb113e9ee2069b058904ced3390c17d1fe36c218", + "modality-request.schema.json": "feb94daa68db1003eb633161ef72431611c1cd5eb6c3096f20c2b69d4a6b9162", + "operation-request.schema.json": "a09fac92af50255d960d1d27ce570c7f89dd377bd7733cead30c4f7fea70a5c5", + "operation-response.schema.json": "17f366d747de6a0047d40a5e549af5abe7045a3a8f64d987c398b7786488d476" }, - "schema_export_command": "PUMAS_IMAGE_CONTRACT_EXPORT_DIR=... ORT_SKIP_DOWNLOAD=1 CARGO_BUILD_JOBS=1 cargo test --locked --offline --manifest-path rust/Cargo.toml -p pumas-rpc --bin pumas-rpc --all-features handlers::model_operations::types::image_to_text_schema_tests::image_to_text_actual_dto_schema_export", - "producer_tree": "cf4d3ca3fbaa77ccd7ba2ebd10c7e9d5ba434ef4", - "schema_export_commit": "5c7d78d7e4eefe65f2395bb3ca2d7bc2ada83c2f" + "schema_export_command": "PUMAS_IMAGE_CONTRACT_EXPORT_DIR=... cargo test --locked -p pumas-rpc --features export-contract image_to_text_actual_dto_schema_export (from rust/)", + "schema_export_commit": "4c79cd8e35db566b70caaa13d8d8f92265f5976e", + "previous_producer": { + "producer_pr": 68, + "producer_commit": "36c2aab2c885b8416fd4e39cf8c37c510596d6e4", + "actual_dto_schema_sha256": { + "capabilities-response.schema.json": "b55b150f6958239e71d71f1b28fd36450907ddccf78544d52b64543ad4f2493d", + "capability-descriptor.schema.json": "eb25783123d38de5ff6aedf4cfa69de8f4f5f5c60799768b1b90da258b85f922", + "error-response.schema.json": "b024363b19d55a004d9577ad04d7e2ec9209d3d1193b7c53c46a7d6ed401e3be", + "modality-request.schema.json": "24b5f0a73a1252d31a3b22687a84f07b16a9d3ea700809a42e615e7f4e668704", + "operation-request.schema.json": "08f46f0e79c6cb288c413b008abbdfbedb1898d02fb98d396ef73faf8634c642", + "operation-response.schema.json": "20bf98dfd7292b1e3a99bf34ec524858351ba6f9d226f17257362e9ddf0500f8" + } + } } diff --git a/tests/test_ai.py b/tests/test_ai.py index dcf601ae..b39c3818 100644 --- a/tests/test_ai.py +++ b/tests/test_ai.py @@ -68,6 +68,13 @@ def test_server_error_explains_missing_projector(self): self.data.suggest(dict(self.body,model='missing-projector')) self.assertIsNone(self.data.sample(self.sample['id'])['annotation']) + def test_pumas_refusal_reason_is_shown_for_any_status(self): + with self.assertRaisesRegex(ai_codex.CodexError, 'HTTP 409.*Pumas reported: model corners-test is served by profiles a, b'): + self.data.suggest(dict(self.body,model='pumas-conflict')) + with self.assertRaises(ai_codex.CodexError) as caught: + self.data.suggest(dict(self.body,model='plain-conflict')) + self.assertNotIn('private upstream wording',str(caught.exception)) + def test_openrouter_catalog_and_auth(self): with patch.object(ai_openrouter,'BASE_URL',self.url): catalog=ai.models(dict(provider='openrouter',api_key='test-key')) diff --git a/tests/test_image_generation.py b/tests/test_image_generation.py index 93e7f74b..576d27e4 100644 --- a/tests/test_image_generation.py +++ b/tests/test_image_generation.py @@ -64,6 +64,23 @@ def test_unprocessable_fields_name_a_contract_mismatch(self): with self.assertRaisesRegex(ValueError, 'contract'): self.manager.generate({**self.body(), 'prompt': 'unprocessable', 'width': 33, 'height': 33}) self.assertEqual(len(self.requests), 1) + def test_pumas_error_message_is_shown_redacted(self): + with self.assertRaises(ValueError) as caught: + self.manager.generate({**self.body(), 'prompt': 'pumas-refusal'}) + message = str(caught.exception) + self.assertIn('Pumas rejected the image request', message) + self.assertIn('Pumas reported: width 33 is not a multiple of 8', message) + self.assertNotIn('sk-abcdefghijkl', message) + with self.assertRaises(ValueError) as caught: + self.manager.generate({**self.body(), 'prompt': 'busy'}) + self.assertNotIn('Pumas reported', str(caught.exception)) + def test_pumas_error_detail_only_reads_pumas_shaped_bodies(self): + import ai_http + self.assertEqual(ai_http.pumas_error_detail(b'{"error":{"message":"m","type":"server_error"}}'), '') + self.assertEqual(ai_http.pumas_error_detail(b''), '') + self.assertEqual(ai_http.pumas_error_detail(b'{"error":{"message":" ","type":"pumas_error"}}'), '') + self.assertEqual(ai_http.pumas_error_detail(b'{"error":{"message":"context window exceeded","type":"pumas_error"}}'), + 'Pumas reported: context window exceeded') def test_no_duration_deadline_remains_and_connect_stays_bounded(self): source = Path(image_generation.__file__).read_text() self.assertFalse(hasattr(image_generation, 'TIMEOUT')) diff --git a/tests/test_pumas_gateways.py b/tests/test_pumas_gateways.py index 2bda674b..7f7d8005 100644 --- a/tests/test_pumas_gateways.py +++ b/tests/test_pumas_gateways.py @@ -57,7 +57,68 @@ def test_exact_named_identity_optional_provenance_and_raw_hash(self): legacy['instance']['build_info'] = None self.assertIsNone(descriptor.decode(json.dumps(legacy).encode())['advertisement']['instance']['build_info']) - def test_versions_boolean_wrong_shapes_unknown_fields_and_oversizing_fail(self): + def test_additive_fields_and_open_list_entries_are_accepted_and_not_retained(self): + additions = [ + lambda v: v.update(future_field={'nested': [1, 2]}), + lambda v: v['instance'].update(connection_token='never retained'), + lambda v: v['instance']['build_info'].update(future_field=True), + lambda v: v['build_info'].update(future_field='ignored'), + lambda v: v['build_info']['protocols'][0].update(future_field='ignored'), + lambda v: v['build_info']['schemas'][0].update(future_field='ignored'), + lambda v: v['build_info']['schemas'].append({'name': 'pumas.discovery', 'version': 2}), + lambda v: v['build_info']['schemas'].append({'name': 'pumas.future-schema', 'version': 1}), + lambda v: v['build_info']['protocols'].append({'name': 'pumas.future-protocol', 'versions': [3]}), + lambda v: v['build_info']['protocols'][-1].update(versions=[1, 2]), + lambda v: v['instance'].update(capabilities=[f'future.capability{i}@1' for i in range(100)]), + ] + for add in additions: + value = fixture(); add(value) + with self.subTest(value=value): + decoded = descriptor.decode(json.dumps(value).encode())['advertisement'] + self.assertEqual(decoded['instance']['registry_library_id'], fixture()['instance']['registry_library_id']) + text = json.dumps(decoded) + self.assertNotIn('future_field', text) + self.assertNotIn('connection_token', text) + value = fixture(); value['future_field'] = 'ignored'; value['instance']['future_field'] = 'ignored' + self.assertEqual(descriptor.decode(json.dumps(value).encode())['advertisement'], fixture()) + endpoint, calls = self.server(value) + self.assertEqual(discovery.inspect_advertised(f'http://127.0.0.1:{endpoint[1]}')['advertisement']['instance'], + fixture()['instance']) + + def test_missing_required_fields_and_unsupported_versions_are_refused_with_reasons(self): + cases = [ + (lambda v: v.pop('service_generation'), + 'Pumas advertisement is missing required field(s) service_generation.'), + (lambda v: v['instance'].pop('generation'), + 'Pumas advertisement instance is missing required field(s) generation.'), + (lambda v: v['build_info'].pop('schemas'), + 'Pumas advertisement build_info is missing required field(s) schemas.'), + (lambda v: v['build_info']['protocols'][0].pop('versions'), + 'Pumas advertisement build_info.protocols[0] is missing required field(s) versions.'), + (lambda v: v.update(advertisement_schema_version=2), + 'Pumas advertisement advertisement_schema_version 2 is unsupported; Tuldok supports version 1.'), + (lambda v: v['instance'].update(selector_schema_version='1'), + 'Pumas advertisement instance.selector_schema_version must be an unsigned 32-bit integer.'), + (lambda v: v.update(endpoint='http://example.com:8765'), + 'Pumas advertisement endpoint is invalid: Enter a numeric-loopback HTTP base URL.'), + ] + for mutate, reason in cases: + value = fixture(); mutate(value) + with self.subTest(reason=reason): + with self.assertRaises(descriptor.Rejected) as caught: + descriptor.decode(json.dumps(value).encode()) + self.assertEqual(str(caught.exception), reason) + value = fixture(); value['instance'].pop('generation') + endpoint, calls = self.server(value) + with patch.object(discovery, 'listening_endpoints', return_value=[endpoint]): + result = discovery.scan_advertised() + self.assertEqual(result['gateways'], []) + self.assertEqual(result['rejected'], [{'server_url': f'http://127.0.0.1:{endpoint[1]}', + 'reason': 'Pumas advertisement instance is missing required field(s) generation.'}]) + with self.assertRaisesRegex(ValueError, 'missing required field'): + discovery.inspect_advertised(f'http://127.0.0.1:{endpoint[1]}') + + def test_versions_boolean_wrong_shapes_and_oversizing_fail(self): mutations = [ lambda v: v.update(advertisement_schema_version=True), lambda v: v.update(advertisement_schema_version=2), @@ -69,8 +130,8 @@ def test_versions_boolean_wrong_shapes_unknown_fields_and_oversizing_fail(self): lambda v: v['build_info']['schemas'][-1].update(version=True), lambda v: v['build_info'].update(compiled_features=[None]), lambda v: v.update(service_generation=''), - lambda v: v['instance'].update(connection_token='do not expose'), - lambda v: v['instance'].update(capabilities=['x'] * 65), + lambda v: v['instance'].update(capabilities='model.query@1'), + lambda v: v.update(instance=[]), lambda v: v['instance'].update(library_root='x' * 4097), lambda v: v.update(endpoint='http://example.com:8765'), lambda v: v.update(endpoint='http://[::1%lo]:8765'), diff --git a/tests/test_pumas_typed.py b/tests/test_pumas_typed.py index b6b76be2..059ddd55 100644 --- a/tests/test_pumas_typed.py +++ b/tests/test_pumas_typed.py @@ -190,18 +190,61 @@ def test_grounded_rewrite_stays_in_source_family_and_draft(self): self.assertEqual(record['review'],'draft'); self.assertEqual(record['parents'],[row['id']]) self.assertEqual(record['groups'],row['groups']); self.assertEqual(record['provenance']['rights'],row['provenance']['rights']) self.assertEqual(record['provenance']['canonical_request']['profile'],'controlled-text-cpu') - def test_manifest_hash_closed_semantics_nonfinite_intervals(self): + def test_manifest_hash_additive_semantics_nonfinite_intervals(self): raw=(FIXTURES/'controlled-text-capabilities.json').read_bytes(); observed=pumas.validate_manifest(raw) self.assertEqual(observed['observed_sha256'],hashlib.sha256(raw).hexdigest()) self.assertEqual(pumas.models(self.url)['models'][0]['id'],'controlled-text') self.state['duplicate_alias']=True self.assertEqual(len(pumas.models(self.url)['models']),2) - for mutate in (lambda v:v.update(extra=1),lambda v:v['capabilities'][0].update(capability={}), - lambda v:v['capabilities'][0]['option_bounds'][0].update(minimum=10**500), + # Additive fields and unrecognised capability entries are ignored, never refused. + for mutate in (lambda v:v.update(extra=1,context_tokens=4096),lambda v:v['capabilities'][0].update(capability={}), + lambda v:v['capabilities'].append({'capability':'decision','semantic_task':'decision'}), + lambda v:v['capabilities'][0]['option_bounds'].append({'option':'top_k','minimum':'n/a'})): + value=copy.deepcopy(observed['capabilities']); mutate(value) + pumas.validate_manifest(json.dumps(value).encode()) + value=copy.deepcopy(observed['capabilities']); value['capabilities'][0].update(capability={}) + with self.assertRaises(pumas.OperationError): + pumas.require(pumas.validate_manifest(json.dumps(value).encode()),'chat_generation','messages_text','text',{'max_tokens':1}) + for mutate in (lambda v:v['capabilities'][0]['option_bounds'][0].update(minimum=10**500), lambda v:v['capabilities'][0]['option_bounds'][0].update(minimum=10,maximum=1), lambda v:v['capabilities'][0].update(availability={'state':'unavailable','reason':{}})): value=copy.deepcopy(observed['capabilities']); mutate(value) with self.assertRaises(ValueError): pumas.validate_manifest(json.dumps(value).encode()) + def test_served_models_listing_profiles_and_fallback(self): + self.state['served_models']={'supported_contract_versions':[1],'max_request_bytes':1,'max_response_bytes':1,'max_stream_event_bytes':1, + 'future':True,'models':[ + {'model':'shared','profile':'gpu','model_id':'lib/a','provider':'llama_cpp','load_state':'loaded','profile_required':True,'capabilities':[],'extra':1}, + {'model':'shared','profile':'cpu','model_id':'lib/a','provider':'llama_cpp','load_state':'loading','profile_required':True,'capabilities':[]}, + {'model':'controlled-text','profile':'controlled-text-cpu','model_id':'lib/t','provider':'ollama','load_state':'loaded','context_tokens':8192, + 'profile_required':False,'capabilities':[]}]} + listed=pumas.models(self.url) + self.assertEqual(listed['source'],'/v1/served-models') + rows={row['id']:row for row in listed['models']} + self.assertEqual(rows['shared']['profiles'],['gpu','cpu']); self.assertTrue(rows['shared']['profile_required']) + self.assertIn('choose one',rows['shared']['name']); self.assertEqual(rows['controlled-text']['name'],'controlled-text') + self.state['served_models']=None + self.assertEqual(pumas.models(self.url)['source'],'/v1/models') + def test_capability_refusals_surface_pumas_message(self): + with self.assertRaises(pumas.OperationError) as caught: pumas.capabilities(self.url,'missing-model') + self.assertIn("no served model is named 'missing-model'",str(caught.exception)); self.assertIn('model_not_found',str(caught.exception)) + self.assertEqual(caught.exception.outcome,'not_admitted') + self.state['ambiguous']=True + with self.assertRaises(pumas.OperationError) as caught: pumas.capabilities(self.url,'controlled-text') + self.assertIn('send profile',str(caught.exception)) + self.state['ambiguous']=False; self.state['mode']='unavailable' + job=self.generate(); self.assertIn('runtime_unavailable',job['error']); self.assertFalse(self.state['requests']) + def test_typed_error_outcome_is_authoritative_and_message_is_retained(self): + for mode,status in (('not_admitted',502),('not_admitted',404),('not_admitted',409),('unknown',400),('unknown',503)): + with self.subTest(mode=mode,status=status): + self.state.update(mode=mode,error_status=status,error_message='provider said '+str(status)) + job=self.generate() + self.assertEqual(job['provider_outcome'],mode); self.assertEqual(job['provider_http_status'],status) + self.assertIn('provider said '+str(status),job['error']) + self.state['mode']='unreadable'; job=self.generate() + self.assertEqual(job['provider_outcome'],'not_admitted'); self.assertIn('not valid JSON',job['error']) + self.state.update(mode='not_admitted',error_status=503,error_message='image runtime is busy') + with self.assertRaises(pumas.OperationError) as caught: self.data.image_requests.generate(self.image()) + self.assertIn('image runtime is busy',str(caught.exception)); self.assertEqual(caught.exception.outcome,'not_admitted') for raw in (b'{"x":1,"x":2}',b'{"x":NaN}',b'{','{"x":1}'.encode('utf-16')): with self.assertRaises(ValueError): pumas.strict_json(raw) diff --git a/tests/test_pumas_vision.py b/tests/test_pumas_vision.py index 22f8a860..91641985 100644 --- a/tests/test_pumas_vision.py +++ b/tests/test_pumas_vision.py @@ -1,4 +1,4 @@ -"""Application acceptance of source-derived vision fixtures and actual DTO schema pins, not inference.""" +"""Application acceptance of source-derived v0.8 vision fixtures and actual DTO schema pins, not inference.""" from auth_helpers import bind_owner import base64 import copy @@ -11,6 +11,7 @@ import uuid from unittest.mock import patch +import jsonschema from PIL import Image from app import Dataset from fake_pumas_vision import FIXTURES, start @@ -39,22 +40,74 @@ def test_actual_dto_schema_identities_and_closed_capability_enum(self): self.assertEqual(hashlib.sha256(raw).hexdigest(), digest, name) self.assertEqual(json.loads(raw)['$schema'], 'https://json-schema.org/draft/2020-12/schema') descriptor = json.loads((FIXTURES/'schemas/capability-descriptor.schema.json').read_text()) - self.assertEqual(set(descriptor['$defs']['Capability']['enum']), set(pumas.CAPABILITIES)) + # Documented capability variants export as `oneOf`; new ones are appended additively. + variants = descriptor['$defs']['Capability']['oneOf'] + names = [name for row in variants for name in row.get('enum', [row.get('const')])] + self.assertTrue(set(pumas.USED_CAPABILITIES) <= set(names)) + self.assertTrue({'multimodal_embedding', 'decision'} <= set(names)) + error = json.loads((FIXTURES/'schemas/error-response.schema.json').read_text()) + self.assertIn('message', error['$defs']['OperationError']['required']) + self.assertEqual(error['properties']['request_id']['type'], ['string', 'null']) for name in ('operation-request', 'modality-request'): schema = json.loads((FIXTURES/'schemas'/(name+'.schema.json')).read_text()) self.assertFalse(schema['additionalProperties']) self.assertEqual(schema['$defs']['ImageEncoding']['enum'], ['png','jpeg']) - def test_seven_source_derived_descriptors_hash_reasons_and_closed_fields(self): + def test_source_derived_descriptors_validate_against_actual_schemas(self): + schemas = {name: json.loads((FIXTURES/'schemas'/(name+'.schema.json')).read_text()) for name in ( + 'capabilities-response', 'operation-request', 'modality-request', 'operation-response', 'error-response')} + def valid(name, value): jsonschema.Draft202012Validator(schemas[name]).validate(value) + valid('capabilities-response', json.loads((FIXTURES/'controlled-capabilities.json').read_bytes())) + valid('operation-request', vision.request(self.job(), image(), facade=False)) + valid('operation-request', vision.request(self.job(), {'kind':'image_messages','messages':[{'role':'user','content':[image()]}]}, facade=False)) + valid('modality-request', {key: value for key, value in vision.request(self.job(), image('jpeg')).items()}) + valid('operation-response', {'contract_version':1,'request_id':'caption-17','result':{'kind':'text','text':'x','finish_reason':'stop'}}) + valid('error-response', {'contract_version':1,'request_id':None,'error':{'code':'invalid_request','outcome':'not_admitted','message':'unreadable body'}}) + def test_source_derived_descriptors_hash_and_select_by_name(self): raw = (FIXTURES/'controlled-capabilities.json').read_bytes() observed = pumas.validate_manifest(raw) self.assertEqual(observed['observed_sha256'],hashlib.sha256(raw).hexdigest()) - self.assertEqual(set(row['capability']for row in observed['capabilities']['capabilities']),set(pumas.CAPABILITIES)) - self.assertEqual(len(pumas.CAPABILITIES),7) - self.assertEqual(observed['capabilities']['capabilities'][4]['option_bounds'][0]['option'],'max_output_tokens') - for mutate in (lambda row:row.update(capability='image_understanding'),lambda row:row['input_formats'].append('image_url'), - lambda row:row['option_bounds'][0].update(maximum=float('inf')),lambda row:row.update(semantic_task='chat_generation')): - value = copy.deepcopy(observed['capabilities']); mutate(value['capabilities'][-1]) - with self.assertRaises(ValueError): pumas.validate_manifest(json.dumps(value).encode()) + names = [row['capability'] for row in observed['capabilities']['capabilities']] + self.assertTrue({'multimodal_embedding','decision'} | set(pumas.USED_CAPABILITIES) <= set(names)) + self.assertEqual(observed['capabilities']['context_tokens'], 8192) + self.assertEqual(pumas.descriptor(observed,'audio_transcription')['option_bounds'][0]['option'],'max_output_tokens') + vision_row = pumas.descriptor(observed,'image_to_text') + self.assertEqual(next(b for b in vision_row['option_bounds'] if b['option']=='max_tokens')['maximum'], 8192) + def test_additive_fields_capabilities_formats_options_and_reasons_are_ignored(self): + value = json.loads((FIXTURES/'controlled-capabilities.json').read_bytes()) + value.update(future_field={'nested':True}, supported_contract_versions=[2,1]) + value['capabilities'].reverse() + value['capabilities'].insert(3, {'capability':'image_understanding','semantic_task':'x','input_formats':[7],'availability':'odd'}) + value['capabilities'].append('not even an object') + row = next(r for r in value['capabilities'] if type(r) is dict and r['capability']=='image_to_text') + row['input_formats'].append('image_url'); row['output_formats'].append('future'); row['future'] = 1 + row['option_bounds'].insert(0, {'option':'future_option','minimum':'unbounded'}) + row['option_bounds'].append({'option':'max_tokens_hint','minimum':1,'maximum':1}) + for item in value['capabilities']: + if type(item) is dict and item.get('capability') == 'decision': + item['availability'] = {'state':'unavailable','reason':'future_reason'} + observed = pumas.validate_manifest(json.dumps(value).encode()) + pumas.require(observed,'image_to_text','messages_image','text',{'max_tokens':4096,'image_count':1}) + with self.assertRaises(pumas.OperationError) as caught: + pumas.require(observed,'image_to_text','messages_image','text',{'max_tokens':8193}) + self.assertIn('8192', str(caught.exception)) + with self.assertRaises(pumas.OperationError) as caught: + pumas.require(observed,'decision','decision','decisions',{}) + self.assertIn('future_reason', str(caught.exception)) + del value['context_tokens']; del value['max_stream_event_bytes'] + pumas.validate_manifest(json.dumps(value).encode()) + def test_used_descriptor_fields_are_still_validated(self): + observed = pumas.validate_manifest((FIXTURES/'controlled-capabilities.json').read_bytes()) + def vision_row(value): return next(r for r in value['capabilities'] if r['capability']=='image_to_text') + for mutate in (lambda v:vision_row(v)['option_bounds'][0].update(maximum=float('inf')), + lambda v:vision_row(v).update(semantic_task='chat_generation'), + lambda v:vision_row(v).update(input_formats='messages_image'), + lambda v:vision_row(v).update(availability={'state':'unavailable','reason':{}}), + lambda v:vision_row(v)['option_bounds'].append({'option':'max_tokens','minimum':1,'maximum':2}), + lambda v:v['capabilities'].append(copy.deepcopy(vision_row(v))), + lambda v:v.update(supported_contract_versions=[2]), + lambda v:v.update(profile=None), lambda v:v.pop('capabilities')): + value = copy.deepcopy(observed['capabilities']); mutate(value) + with self.subTest(value=value), self.assertRaises(ValueError): pumas.validate_manifest(json.dumps(value).encode()) def test_named_and_facade_image_order_and_original_compressed_bytes(self): for encoding in ('png','jpeg'): part = image(encoding,progressive=encoding=='jpeg') @@ -98,18 +151,40 @@ def messages(parts,role='user'):return {'kind':'messages','messages':[{'role':ro with self.assertRaises(ValueError): vision.request(self.job(),value) self.assertEqual((vision.MAX_IMAGE,vision.MAX_AGGREGATE,vision.MAX_REQUEST,vision.MAX_IMAGES),(8388608,16777216,33554432,4)) def test_finite_options_and_correlated_typed_result_without_model(self): - for key,value in [('max_tokens',0),('max_tokens',2049),('max_tokens',True),('temperature',float('nan')), + for key,value in [('max_tokens',0),('max_tokens',2**32),('max_tokens',True),('temperature',float('nan')), ('temperature',3),('top_p',float('inf')),('top_p',-1)]: with self.assertRaises(ValueError): vision.request(self.job(),image(),**{key:value}) - request=vision.request(self.job(),image(),max_tokens=2048,temperature=2,top_p=1) - self.assertEqual(request['options'],{'kind':'text_generation','max_tokens':2048,'temperature':2,'top_p':1}) + # No fixed 2048 cap: the live max_tokens bound follows the served context. + request=vision.request(self.job(),image(),max_tokens=8192,temperature=2,top_p=1) + self.assertEqual(request['options'],{'kind':'text_generation','max_tokens':8192,'temperature':2,'top_p':1}) response={'contract_version':1,'request_id':'caption-17','result':{'kind':'text','text':'A blue rectangle.','finish_reason':'stop'}} self.assertEqual(pumas.result(encode(response).encode(),'caption-17')['text'],'A blue rectangle.') - for mutate in (lambda v:v.update(request_id='foreign'),lambda v:v.update(model='private'), - lambda v:v['result'].update(finish_reason='length'),lambda v:v['result'].update(extra='private')): + for mutate in (lambda v:v.update(model='additive'),lambda v:v['result'].update(extra='additive')): + value=copy.deepcopy(response);mutate(value) + self.assertEqual(pumas.result(encode(value).encode(),'caption-17')['text'],'A blue rectangle.') + for mutate in (lambda v:v.update(request_id='foreign'),lambda v:v['result'].update(finish_reason='length'), + lambda v:v.update(contract_version=2),lambda v:v['result'].update(kind='decisions')): value=copy.deepcopy(response);mutate(value) with self.assertRaises(pumas.OperationError)as caught:pumas.result(encode(value).encode(),'caption-17') self.assertEqual(caught.exception.outcome,'unknown') + def test_typed_errors_use_outcome_message_and_null_request_id(self): + def error(request_id='caption-17', **fields): + body={'code':'model_not_found','outcome':'not_admitted','message':'no served model is named "x"'}; body.update(fields) + return encode({'contract_version':1,'request_id':request_id,'error':body}).encode() + cases=[(error(),'not_admitted','model_not_found'),(error(None,code='invalid_request'),'not_admitted','invalid_request'), + (error(code='ambiguous_operation'),'not_admitted','ambiguous_operation'), + (error(code='future_code'),'not_admitted','future_code'), + (error(code='provider_failure',outcome='unknown'),'unknown','provider_failure'), + (error(outcome='future_outcome'),'unknown','model_not_found')] + for raw,outcome,code in cases: + with self.subTest(raw=raw),self.assertRaises(pumas.OperationError)as caught:pumas.result(raw,'caption-17') + self.assertEqual((caught.exception.outcome,caught.exception.provider_code),(outcome,code)) + self.assertIn('no served model is named "x"',str(caught.exception)) + for raw in (error('foreign'),error(code=7),encode({'contract_version':1,'request_id':'caption-17','error':'text'}).encode()): + with self.subTest(raw=raw),self.assertRaises(pumas.OperationError)as caught:pumas.result(raw,'caption-17') + self.assertEqual(caught.exception.outcome,'unknown') + with self.assertRaises(pumas.OperationError)as caught:pumas.result(error(message='x'*5000+'\nnext'),'caption-17') + self.assertLess(len(str(caught.exception)),800);self.assertNotIn('\n',str(caught.exception)) class VisionApplicationTests(unittest.TestCase): @@ -151,16 +226,24 @@ def test_build_selection_and_unavailability_refuse_without_post(self): for mode in ('missing_build','unavailable','wrong_profile','wrong_model'): self.state['mode']=mode;job=self.generate() self.assertEqual(job['status'],'failed');self.assertEqual(job['provider_outcome'],'not_admitted') + if mode=='unavailable':self.assertIn('runtime_unavailable',job['error']) self.assertFalse(self.state['requests']) with self.assertRaises(ValueError):self.data.caption_proposals.start(self.body(seed=42)) def test_http_refusals_outcomes_and_uncertain_failures_are_never_replayed(self): - for status in (400,413,422,503,502): - mode='unknown'if status==502 else'not_admitted' - self.state.update(mode=mode,refusal_status=status);body=self.body();job=self.generate(body) - self.assertEqual(job['provider_outcome'],mode);self.assertEqual(job['provider_http_status'],status) + # `outcome` is authoritative, not the HTTP status (404/409 are ordinary refusals). + cases=[(400,'not_admitted'),(404,'not_admitted'),(409,'not_admitted'),(413,'not_admitted'),(422,'not_admitted'), + (503,'not_admitted'),(502,'not_admitted'),(502,'unknown'),(500,'unknown'),(400,'unknown')] + for status,mode in cases: + self.state.update(mode=mode,refusal_status=status,error_message='reason for '+str(status));body=self.body();job=self.generate(body) + self.assertEqual(job['provider_outcome'],mode,status);self.assertEqual(job['provider_http_status'],status) + self.assertIn('reason for '+str(status),job['error']) self.data.caption_proposals.start(body) - self.assertEqual(len(self.state['requests']),5) - for mode in ('wrong_id','private','length','nonfinite','oversized','truncated','loss'): + self.state['mode']='unreadable';job=self.generate() + self.assertEqual(job['provider_outcome'],'not_admitted');self.assertEqual(job['provider_error_code'],'invalid_request') + self.assertIn('request body is not valid JSON',job['error']) + self.assertEqual(len(self.state['requests']),len(cases)+1) + self.state['mode']='additive';self.assertEqual(self.generate()['status'],'completed') + for mode in ('wrong_id','length','nonfinite','oversized','truncated','loss'): self.state['mode']=mode;body=self.body();before=len(self.state['requests']);job=self.generate(body) self.assertEqual(job['status'],'failed',mode);self.assertEqual(job['provider_outcome'],'unknown',mode) self.data.caption_proposals.start(body);self.assertEqual(len(self.state['requests']),before+1)