From 6f2a6926d8815ba5a72e27d8cf88155df65ef857 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 2 Nov 2026 09:59:49 -0700 Subject: [PATCH 1/3] fix(pumas): consume the v0.8 additive typed HTTP contract Pumas v0.8 (feat/v080-consumer-api, 4c79cd8e) grows typed responses additively and makes error.outcome authoritative. Tuldok refused valid responses and misreported refusals. - Capabilities: validate only the fields, capabilities (chat, image generation, image-to-text) and option bounds Tuldok uses; ignore context_tokens, multimodal_embedding/decision, new formats, reasons, options and fields; select by name; drop list-length caps (1 MiB body bound instead). - Errors: classify by error.outcome, not HTTP status (404/409 are ordinary refusals); accept request_id null; keep unknown codes; show Pumas' bounded error.message in job/UI errors, capability refusals and pumas_error bodies on OpenAI-compatible routes (after redaction). - Results: ignore additive envelope/result fields. - Image-to-text: no fixed 2048 max_tokens cap; the live bound follows the served context. Image facts are checked only when advertised. - Model listing prefers GET /v1/served-models (profiles shown), falling back to /v1/models on 404. - Repin the six actual DTO schema exports, source-derived capability fixture and source.json to 4c79cd8e; update the consumer contract doc. The PR51 advertisement decoder stays closed because it is retained as evidence. Co-Authored-By: Claude Opus 5.5 --- ai_http.py | 42 ++- .../contracts/pumas-image-to-text-consumer.md | 103 ++++-- gateway_discovery.py | 10 +- image_generation.py | 8 +- pumas_operations.py | 324 +++++++++++++----- pumas_vision.py | 15 +- tests/fake_ai.py | 6 + tests/fake_images.py | 3 + tests/fake_pumas_typed.py | 14 +- tests/fake_pumas_vision.py | 19 +- .../controlled-capabilities.json | 126 +++++-- .../schemas/capabilities-response.schema.json | 54 ++- .../schemas/capability-descriptor.schema.json | 45 ++- .../schemas/error-response.schema.json | 7 +- .../schemas/modality-request.schema.json | 256 +++++++++++++- .../schemas/operation-request.schema.json | 167 ++++++++- .../schemas/operation-response.schema.json | 117 +++++++ .../pumas-image-to-text-v1/source.json | 57 +-- tests/test_ai.py | 7 + tests/test_image_generation.py | 17 + tests/test_pumas_typed.py | 49 ++- tests/test_pumas_vision.py | 125 +++++-- 22 files changed, 1315 insertions(+), 256 deletions(-) diff --git a/ai_http.py b/ai_http.py index c1ea21bf..d1ac25ac 100644 --- a/ai_http.py +++ b/ai_http.py @@ -88,6 +88,26 @@ def collect(value, depth=0): return 'Provider detail: ' + ' | '.join(messages)[:340] if messages else 'The provider returned no readable error message.' +def pumas_error_detail(data, payload=None, headers=None): + """Pumas' OpenAI-compatible errors carry its (redacted) reason in `error.message`. + + Returns '' for any body that is not a bounded `pumas_error`; that reason is + still passed through the same redaction as other provider diagnostics. + """ + if not data or len(data) > MAX_ERROR_BODY: + return '' + try: + value = json.loads(data) + except (ValueError, UnicodeError): + return '' + error = value.get('error') if isinstance(value, dict) else None + if (not isinstance(error, dict) or error.get('type') != 'pumas_error' + or not isinstance(error.get('message'), str) or not error['message'].strip()): + return '' + detail = provider_error_detail(json.dumps({'message': error['message']}).encode(), payload, headers) + return 'Pumas reported: ' + detail.removeprefix('Provider detail: ') if detail.startswith('Provider detail: ') else '' + + def request_error_hint(data): """Classify bounded provider errors without exposing arbitrary response text.""" try: @@ -184,18 +204,24 @@ def watch_cancel(): 403: 'Check account permissions and provider restrictions.', 429: 'Rate limit reached; wait before trying again.'}.get(response.status, 'Check the selected model and structured-output support.') - if response.status == 400 or response.status >= 500: - try: - transport.settimeout(max(.001, min(2, deadline - time.monotonic()))) - error_data = response.read(MAX_ERROR_BODY + 1) - specific = request_error_hint(error_data) if len(error_data) <= MAX_ERROR_BODY else '' + try: + transport.settimeout(max(.001, min(2, deadline - time.monotonic()))) + error_data = response.read(MAX_ERROR_BODY + 1) + except (OSError, http.client.HTTPException): + error_data = None + if label == 'OpenRouter' and (response.status == 400 or response.status >= 500): + hint = 'Could not read the provider error body within the diagnostic limit.' + if error_data is not None: + specific = request_error_hint(error_data) if len(error_data) <= MAX_ERROR_BODY else '' + pumas_detail = pumas_error_detail(error_data, payload, headers) + if pumas_detail: + # Pumas' own (redacted) reason is shown for every refusal status. + hint = (specific + ' ' if specific else '') + pumas_detail + elif response.status == 400 or response.status >= 500: if label == 'OpenRouter' or response.status >= 500: hint = provider_error_detail(error_data, payload, headers) elif specific: hint = specific - except (OSError, http.client.HTTPException): - if label == 'OpenRouter': - hint = 'Could not read the provider error body within the diagnostic limit.' raise CodexError(f'{label} returned HTTP {response.status}. {hint}', retryable=response.status in (502, 503, 504)) yield response, transport, deadline diff --git a/docs/contracts/pumas-image-to-text-consumer.md b/docs/contracts/pumas-image-to-text-consumer.md index a6fa3697..287693cf 100644 --- a/docs/contracts/pumas-image-to-text-consumer.md +++ b/docs/contracts/pumas-image-to-text-consumer.md @@ -1,22 +1,47 @@ # Pumas Image→Text consumer -This opt-in caption path consumes the contract at Pumas Library merged PR68, -source `36c2aab2c885b8416fd4e39cf8c37c510596d6e4`: -[Image→Text contract](https://github.com/MrScripty/Pumas-Library/blob/36c2aab2c885b8416fd4e39cf8c37c510596d6e4/docs/contracts/image-to-text-v0.8.md). -It is based on Tuldok main `d99e23d2b0d2783a7a9863232a7453cd9f2db7aa`. -The legacy compatible caption path and the previous typed text/image operations -remain available. Their existing producer receipt is not silently repinned. +This opt-in caption path, and the typed text and image operations that share +`pumas_operations`, consume the Pumas Library v0.8 typed HTTP contract on branch +`feat/v080-consumer-api`, source `4c79cd8e35db566b70caaa13d8d8f92265f5976e`: +[model operations, contract version 1](https://github.com/MrScripty/Pumas-Library/blob/4c79cd8e35db566b70caaa13d8d8f92265f5976e/docs/contracts/model-operations-v1.md). +It supersedes the merged PR68 pin `36c2aab2c885b8416fd4e39cf8c37c510596d6e4`, +which `source.json` keeps as `previous_producer`. The legacy compatible caption +path remains available. The PR54 typed text/image capture receipt in +`tests/fixtures/pumas-typed-v1` is historical and is not repinned. -`pumas_operations.validate_manifest` accepts all seven closed capability variants, -the three additional image input formats and image byte/pixel/count option names. -The existing unavailable reasons remain unchanged. Availability is a live -observation, not a model qualification receipt. Audio's existing -`max_output_tokens` descriptor option is accepted as declared in the inspected DTO. +### Additive compatibility + +Pumas responses grow additively within contract version 1. Tuldok validates +only what it uses and selects capabilities and option bounds by name, never by +position or count: + +- `GET /v1/capabilities` requires `supported_contract_versions` containing 1, + `model`, an exact `profile`, positive `max_request_bytes`/`max_response_bytes` + and a `capabilities` list. Other fields (`context_tokens`, + `max_stream_event_bytes`, future fields) are retained but not required. +- Only `chat_generation`, `image_generation` and `image_to_text` descriptors are + validated: fixed semantic task, string format lists, an availability `state` + with an optional string reason, and finite, ordered bounds for the options + Tuldok compares (`max_tokens`, `temperature`, `top_p`, `width`, `height`, + `seed`, `image_bytes`, `image_pixels`, `image_count`). A used capability or + option advertised twice is refused as ambiguous. +- Other capabilities (`multimodal_embedding`, `decision`, audio, future values), + formats (`content_batch`, `decision`, `decisions`, future values), option names, + availability states and reasons, and response or result fields are ignored. + Any availability other than `available` means unavailable, and its reason is + shown. +- The capability observation is bounded at 1 MiB. `GET /v1/served-models` is + preferred for listing serving aliases with their profiles; gateways without it + (HTTP 404) fall back to `/v1/models`, which lists each ID once with `profiles`. + +Availability is a live observation, not a model qualification receipt. Choose **Pumas typed v1** in the image caption form, select the serving alias and optional exact profile, and inspect capabilities. On each new attempt, the owner requires `pumas.model-operations.image-to-text:1` in the gateway build advertisement -and an available Image→Text descriptor for the exact resolved model/profile. +and an available Image→Text descriptor for the exact resolved model/profile. The +gateway advertisement keeps its own closed, versioned decoder: it is retained as +evidence, so unknown advertisement fields are refused rather than stored. The request uses the facade's `messages`, `semantic_task:"image_to_text"`, text-generation options and `stream:false`. The seed is omitted. Named image and `image_messages` inputs are also supported by the bounded consumer request builder. @@ -25,8 +50,10 @@ messages can contain images; audio, URLs, paths and data-URL envelopes are refus The consumer applies the producer's 32 MiB whole-request, 8 MiB per-image, 16 MiB aggregate, four-image, 4096-axis, 4,194,304-pixel, 128-message and 128-total-part -bounds. Tokens default to 512 and permit 1–2048; temperature/top-p must be finite -and within 0–2/0–1. The caption application's existing preparation is smaller: +bounds. Tokens default to 512. Pumas no longer caps Image→Text `max_tokens` at +2048; the live `max_tokens` bound follows the served context window and is +checked before admission. Temperature/top-p must be finite and within 0–2/0–1. +Advertised image byte, pixel, count and dimension bounds are checked when present. The caption application's existing preparation is smaller: one RGB JPEG, at most 2 MiB and 1600 pixels on its longest side. It decodes the captured, hash-verified canonical image and retains both canonical-source and prepared-input hashes. Preparation is explicit in request history. @@ -37,8 +64,8 @@ replace Pumas's production PNG/turbojpeg validation, allocation controls or admission authority, and does not claim equivalent JPEG warning detection. No local validation result is used as permission to bypass provider refusals. -The response is the closed `{contract_version, request_id, result}` envelope. -The request ID must match; `result.kind` must be text. There is no response-model +The response is the `{contract_version, request_id, result}` envelope; added +fields are ignored. The request ID must match; `result.kind` must be text. There is no response-model requirement and no private provider envelope projection. This application requires a complete `stop` finish reason and valid caption JSON within a 256 KiB response bound; partial or filtered output is not applied. Capability/build observations, @@ -47,8 +74,15 @@ on the exact job. List responses omit image-bearing canonical requests and raw bytes. Explicit application creates a draft target and records its evidence in revision history; human review remains separate. -HTTP 400/413/422/503 pre-admission refusals and 502 failures retain their typed -code, HTTP status and `outcome` in the asynchronous job. A disconnect, uncertain +Every typed error is `{contract_version, request_id, error:{code, outcome, message}}`. +`error.outcome` is authoritative: `not_admitted` records no provider effect and +anything else is `unknown`, whatever the HTTP status. 404 `model_not_found` and +409 `ambiguous_model`/`ambiguous_operation` are ordinary refusals. An error whose +`request_id` is `null` (Pumas could not read the body) is still a typed refusal. +Unknown error codes are kept verbatim. The job retains the code, HTTP status and +outcome, and its error text includes Pumas' `error.message`, bounded and on one +line. Capability refusals show the same message. On the OpenAI-compatible routes, +a `pumas_error` body's `error.message` is shown after the existing redaction. A disconnect, uncertain response or possibly dispatched interrupted attempt remains `unknown`. Repeating an existing exact application request ID returns the retained attempt; it never replays inference. A new attempt requires a new explicit request. Browser recovery @@ -65,27 +99,30 @@ and packaged application acceptance remain integration-lead/provider-owner gates `tests/fixtures/pumas-image-to-text-v1/source.json` labels the capability/wire fixture as **source-derived**, with inspected production-source hashes. It is not -a captured native DTO schema export or an actual model reply. The tests exercise -seven-capability parsing, named/facade PNG/JPEG inputs (including progressive +a captured native DTO schema export or an actual model reply. The fixture follows +the pinned descriptor construction for a llama.cpp vision serving with an +8192-token context, including the `multimodal_embedding` and `decision` +descriptors. The tests exercise additive capability parsing, named/facade PNG/JPEG inputs (including progressive JPEG), order, closed shapes, finite/resource bounds, typed application captions, draft/history, live selection refusal, cancellation/borrowed-owner survival, HTTP outcomes, restart and no replay. The existing typed browser suite additionally checks the actual caption form, capability inspection, application and disconnect. -The six actual production DTO schemas regenerated from the combined source -are retained in `tests/fixtures/pumas-image-to-text-v1/schemas/`, with SHA-256 -identities in `source.json`. The source-derived capability fixture and authored -named/facade image requests and typed responses validate against these exports. -The schemas describe wire shape; finite bounds and live readiness remain runtime -checks. Neither fixture output nor schema validity proves native inference. +The six actual production DTO schemas regenerated at the pinned source with the +`export-contract` feature are retained in +`tests/fixtures/pumas-image-to-text-v1/schemas/`, with SHA-256 identities in +`source.json`. All six changed from the PR68 export (new capabilities, formats, +`context_tokens`, decision and content-batch DTOs and the required error +`message`). The +source-derived capability fixture and authored named/facade image requests and +typed responses validate against these exports in the vision tests. The schemas +describe wire shape; finite bounds and live readiness remain runtime checks. +Neither fixture output nor schema validity proves native inference. -The reviewed producer retains identical DTO definitions and schema dependencies; -all six exported schema SHA-256 identities are unchanged. `source.json` records -the export commit and refreshed handler/controlled-test source hashes. The fixture -gateway advertisement reads the same versioned source pin as its consumer -acceptance evidence. -This pin is the immutable merged Pumas main commit. Its source tree is identical -to the reviewed producer that passed CI before the coordinated merge. +`source.json` records the branch, commit, tree, contract document hash and +handler/controlled-test source hashes. The pin is a branch commit, not a merged +Pumas main commit; repin when it merges. The fixture gateway advertisement reads +the same versioned source pin as its consumer acceptance evidence. The combined desktop contract and packaged application are provider integration gates. Real model/projector inference, semantic quality, managed native teardown and production audio qualification remain unclaimed. No models or runtimes are diff --git a/gateway_discovery.py b/gateway_discovery.py index 314023d3..db3afc8b 100644 --- a/gateway_discovery.py +++ b/gateway_discovery.py @@ -35,7 +35,10 @@ def listening_endpoints(): return sorted(endpoints, key=lambda endpoint: (endpoint[1], endpoint[0])) -def get_json(host, port, path, deadline, payload=None, max_response=MAX_RESPONSE, decode=json.loads, require_complete=False): +def get_json(host, port, path, deadline, payload=None, max_response=MAX_RESPONSE, decode=json.loads, require_complete=False, + error=None): + """GET/POST bounded JSON. With `error`, a non-200 body is read under the same bounds + and passed to `error(status, raw)`, which raises a readable refusal.""" timeout = min(1.5, deadline - time.monotonic()) if timeout <= 0: raise TimeoutError() @@ -64,7 +67,7 @@ def expire(): expiry.daemon = True expiry.start() with connection.getresponse() as response: - if response.status != 200: + if response.status != 200 and error is None: raise ValueError('Not a gateway response') if require_complete: declared = response.getheader('Content-Length') @@ -86,6 +89,9 @@ def expire(): raise ValueError('Pumas descriptor response ended before its declared length') if require_complete and time.monotonic() >= deadline: raise TimeoutError() + if response.status != 200: + error(response.status, bytes(data)) + raise ValueError('Not a gateway response') return decode(data) finally: if expiry is not None: diff --git a/image_generation.py b/image_generation.py index 76e37f2f..2ca126f5 100644 --- a/image_generation.py +++ b/image_generation.py @@ -258,7 +258,13 @@ def watch_cancel(): 499: 'Image generation cancelled.', 503: 'The image model is unavailable. Load it in Pumas and refresh models.', 507: 'Pumas reported insufficient GPU memory. Free memory before trying again.'} - raise ValueError(messages.get(response.status, 'Pumas image generation failed. Check its runtime status. The request was not retried.')) + message = messages.get(response.status, 'Pumas image generation failed. Check its runtime status. The request was not retried.') + try: + transport.settimeout(2) + detail = ai_http.pumas_error_detail(response.read(ai_http.MAX_ERROR_BODY + 1), payload) + except (OSError, http.client.HTTPException): + detail = '' + raise ValueError(message + (' ' + detail if detail else '')) data = bytearray() while not response.isclosed(): chunk = response.read1(65536) diff --git a/pumas_operations.py b/pumas_operations.py index 89d345bc..3878473a 100644 --- a/pumas_operations.py +++ b/pumas_operations.py @@ -1,4 +1,9 @@ -"""Finite PR54 selected-model operations. Discovery budgets never time generation.""" +"""Finite Pumas typed model operations (contract version 1, v0.8 additive rules). + +Discovery budgets never time generation. Responses grow additively: unknown +response fields, capabilities, formats, option names, availability reasons and +error codes are ignored; only what Tuldok uses is validated and selected by name. +""" import hashlib import http.client import json @@ -13,20 +18,20 @@ from workbench import WorkbenchError, encode PROTOCOL = 'pumas_typed_v1' -SOURCE_COMMIT = '40c5cbfed67a6f0e862a1197bb5105363d67bdb1' -MAX_CAPABILITIES = 65536 +SOURCE_COMMIT = '4c79cd8e35db566b70caaa13d8d8f92265f5976e' +MAX_CAPABILITIES = 1024 * 1024 +MAX_SERVED_MODELS = 8 * 1024 * 1024 MAX_TEXT_RESPONSE = 256 * 1024 MAX_IMAGE_JSON = 12 * 1024 * 1024 MAX_DIMENSION = 2048 MAX_PIXELS = 4194304 -CAPABILITIES = {'chat_generation': 'chat_generation', 'text_generation': 'text_generation', - 'text_embedding': 'text_embedding', 'image_generation': 'text_to_image', - 'image_to_text': 'image_to_text', - 'audio_transcription': 'speech_to_text', 'audio_classification': 'audio_classification'} -ERROR_CODES = {'invalid_request', 'unsupported_contract', 'model_not_found', 'ambiguous_model', - 'ambiguous_operation', 'unsupported_modality', - 'capability_unavailable', 'provider_failure', 'invalid_provider_result', - 'request_limit', 'response_limit', 'transport_lost'} +MAX_MESSAGE = 600 +# Capabilities Tuldok sends, with their fixed contract-version-1 semantic task. +USED_CAPABILITIES = {'chat_generation': 'chat_generation', 'image_generation': 'text_to_image', + 'image_to_text': 'image_to_text'} +# Option bounds Tuldok compares against. Other advertised options are ignored. +USED_OPTIONS = {'max_tokens', 'temperature', 'top_p', 'width', 'height', 'seed', + 'image_bytes', 'image_pixels', 'image_count'} class OperationError(WorkbenchError): @@ -56,10 +61,63 @@ def constant(_): def object_fields(value, required, optional=()): + """Closed shape for request-side DTOs that Tuldok itself builds.""" if type(value) is not dict or not set(required) <= value.keys() or value.keys() - set(required) - set(optional): raise ValueError('Unsupported typed Pumas fields.') +def response_fields(value, required): + """Open shape for Pumas responses: required fields present, additive fields ignored.""" + if type(value) is not dict or not set(required) <= value.keys(): + raise ValueError('Typed Pumas response is missing required fields.') + + +def _positive_int(value): + return type(value) is int and value > 0 + + +def message_text(value): + """Bounded single-line display of Pumas' human-readable reason (never parsed).""" + if not isinstance(value, str): + return '' + value = ' '.join(''.join(c if c.isprintable() else ' ' for c in value).split()) + return value[:MAX_MESSAGE] + ('…' if len(value) > MAX_MESSAGE else '') + + +def typed_error(value): + """Decode `{contract_version, request_id, error:{code, outcome, message}}` or return None. + + `outcome` is authoritative. Only `not_admitted` proves no provider effect; + any other value is treated as `unknown`. Unknown codes are kept verbatim. + """ + if type(value) is not dict or type(value.get('error')) is not dict: + return None + error = value['error'] + code = error.get('code') + if not isinstance(code, str) or not code or len(code) > 200: + return None + outcome = 'not_admitted' if error.get('outcome') == 'not_admitted' else 'unknown' + return {'code': code, 'outcome': outcome, 'message': message_text(error.get('message'))} + + +def error_message(error, prefix='Pumas'): + detail = (': ' + error['message']) if error['message'] else '.' + if error['outcome'] == 'not_admitted': + return f"{prefix} refused the request ({error['code']}; not admitted, no provider effect){detail}" + return f"{prefix} reported {error['code']}; provider outcome unknown, no automatic replay{detail}" + + +def http_error(status, raw): + """Raise a readable ValueError for a non-200 typed discovery response.""" + try: + error = typed_error(strict_json(raw)) + except ValueError: + error = None + if error is None: + raise ValueError(f'Pumas returned HTTP {status} without a typed error body.') + raise ValueError(error_message(error)) + + def alias(value): if (not isinstance(value, str) or not value or value != value.strip() or len(value.encode('utf-8')) > 256 or any(ord(c) < 32 or ord(c) == 127 for c in value)): @@ -89,50 +147,69 @@ def configuration(body, *, image=False): return {'protocol': PROTOCOL, 'profile': profile(body['profile'])} +def _bound_number(value): + return type(value) in (int, float) and -1e308 <= value <= 1e308 + + +def _descriptor(item): + """Validate one descriptor Tuldok uses. Extra fields and enum values are ignored.""" + response_fields(item, ('capability', 'semantic_task', 'input_formats', 'output_formats', 'availability', 'option_bounds')) + name = item['capability'] + if item['semantic_task'] != USED_CAPABILITIES[name]: + raise ValueError('Pumas reports a different semantic task for '+name+' within contract version 1.') + for field in ('input_formats', 'output_formats'): + if type(item[field]) is not list or any(type(value) is not str for value in item[field]): + raise ValueError('Invalid typed Pumas '+field.replace('_', ' ')+'.') + availability = item['availability'] + if (type(availability) is not dict or type(availability.get('state')) is not str + or availability.get('reason') is not None and type(availability['reason']) is not str): + raise ValueError('Invalid typed Pumas availability.') + bounds = item['option_bounds'] + if type(bounds) is not list: + raise ValueError('Invalid typed Pumas option bounds.') + seen = set() + for bound in bounds: + if type(bound) is not dict or type(bound.get('option')) is not str or bound['option'] not in USED_OPTIONS: + continue + response_fields(bound, ('option', 'minimum', 'maximum')) + if bound['option'] in seen: + raise ValueError('Pumas advertises the '+bound['option']+' bound more than once.') + seen.add(bound['option']) + if not _bound_number(bound['minimum']) or not _bound_number(bound['maximum']) or bound['minimum'] > bound['maximum']: + raise ValueError('Invalid typed Pumas option interval.') + + def validate_manifest(raw): + """Validate the fields Tuldok uses from `GET /v1/capabilities`. + + Additive fields (for example `context_tokens`), unused capabilities such as + `multimodal_embedding` or `decision`, and unknown formats, reasons and option + names are accepted and ignored. The raw observation is retained unchanged. + """ value = strict_json(raw) - object_fields(value, ('supported_contract_versions', 'model', 'profile', 'max_request_bytes', - 'max_response_bytes', 'max_stream_event_bytes', 'capabilities')) + response_fields(value, ('supported_contract_versions', 'model', 'profile', 'max_request_bytes', + 'max_response_bytes', 'capabilities')) versions = value['supported_contract_versions'] - if type(versions) is not list or not 1 <= len(versions) <= 16 or any(type(v) is not int for v in versions) or 1 not in versions: + if type(versions) is not list or not any(type(version) is int and version == 1 for version in versions): raise ValueError('Pumas typed contract version 1 is unavailable.') alias(value['model']) if profile(value['profile']) is None: raise ValueError('Pumas must resolve an exact profile.') - for key, maximum in [('max_request_bytes', 32*1024*1024), ('max_response_bytes', 32*1024*1024), ('max_stream_event_bytes', 256*1024)]: - if type(value[key]) is not int or not 0 < value[key] <= maximum: + for key in ('max_request_bytes', 'max_response_bytes'): + if not _positive_int(value[key]): raise ValueError('Unsupported typed Pumas byte bounds.') entries = value['capabilities'] - if type(entries) is not list or len(entries) > len(CAPABILITIES): + if type(entries) is not list: raise ValueError('Invalid typed Pumas capability list.') seen = set() for item in entries: - object_fields(item, ('capability', 'semantic_task', 'input_formats', 'output_formats', 'streaming', 'availability', 'option_bounds')) - name = item['capability'] - if type(name) is not str or name not in CAPABILITIES or name in seen or item['semantic_task'] != CAPABILITIES[name] or type(item['streaming']) is not bool: - raise ValueError('Unsupported typed Pumas semantic capability.') + name = item.get('capability') if type(item) is dict else None + if type(name) is not str or name not in USED_CAPABILITIES: + continue + if name in seen: + raise ValueError('Pumas advertises '+name+' more than once for one serving.') seen.add(name) - for field, allowed in [('input_formats', {'messages_text', 'text', 'text_batch', 'pcm_s16le', 'pcm_f32le', 'png_base64', 'jpeg_base64', 'messages_image'}), - ('output_formats', {'text', 'embeddings_float32', 'png_base64', 'labels'})]: - items = item[field] - if type(items) is not list or not 1 <= len(items) <= len(allowed) or any(type(v) is not str or v not in allowed for v in items) or len(set(items)) != len(items): - raise ValueError('Unsupported typed Pumas format.') - availability = item['availability'] - object_fields(availability, ('state',), ('reason',)) - if availability not in [{'state': 'available'}] and (availability.get('state') != 'unavailable' or - type(availability.get('reason')) is not str or availability.get('reason') not in {'unsupported_adapter', 'unqualified_audio_runtime', 'unknown_model_task', 'model_task_mismatch', 'runtime_unavailable'}): - raise ValueError('Invalid typed Pumas availability.') - bounds = item['option_bounds'] - if type(bounds) is not list or len(bounds) > 13: - raise ValueError('Invalid typed Pumas option bounds.') - options = set() - for bound in bounds: - object_fields(bound, ('option', 'minimum', 'maximum')) - if type(bound['option']) is not str or bound['option'] not in {'max_tokens', 'temperature', 'top_p', 'dimensions', 'input_count', 'input_characters', 'width', 'height', 'seed', 'max_output_tokens', 'image_bytes', 'image_pixels', 'image_count'} or bound['option'] in options: - raise ValueError('Unsupported typed Pumas option.') - options.add(bound['option']) - if any(type(bound[k]) not in (int, float) or not -1e308 <= bound[k] <= 1e308 for k in ('minimum', 'maximum')) or bound['minimum'] > bound['maximum']: - raise ValueError('Invalid typed Pumas option interval.') + _descriptor(item) return {'capabilities': value, 'observed_sha256': hashlib.sha256(raw).hexdigest()} @@ -144,7 +221,8 @@ def capabilities(base, model, exact_profile=None): query['profile'] = exact_profile try: observed = gateway_discovery.get_json(parts.hostname, parts.port, '/v1/capabilities?' + urlencode(query), - time.monotonic()+3, max_response=MAX_CAPABILITIES, decode=validate_manifest, require_complete=True) + time.monotonic()+3, max_response=MAX_CAPABILITIES, decode=validate_manifest, require_complete=True, + error=http_error) except (ValueError, OSError, http.client.HTTPException) as error: raise OperationError('Selected-model capabilities unavailable: '+str(error)) from None manifest = observed['capabilities'] @@ -153,37 +231,102 @@ def capabilities(base, model, exact_profile=None): return observed +class _RouteMissing(ValueError): + pass + + +def _listing(aliases): + rows = [] + for name, profiles in aliases.items(): + required = len(profiles) > 1 + label = name + (' (profiles: ' + ', '.join(profiles) + '; choose one)' if required else '') + rows.append({'id': name, 'name': label, 'profiles': profiles, 'profile_required': required}) + return rows + + +def _served_models(raw): + """`GET /v1/served-models`: one entry per served (model, profile) pair.""" + value = strict_json(raw) + response_fields(value, ('models',)) + if type(value['models']) is not list: + raise ValueError('Invalid Pumas served-model listing.') + aliases = {} + for item in value['models']: + response_fields(item, ('model', 'profile')) + selected = profile(item['profile']) + if selected is None: + raise ValueError('Pumas served-model entry has no exact profile.') + profiles = aliases.setdefault(alias(item['model']), []) + if selected not in profiles: + profiles.append(selected) + return {'models': _listing(aliases), 'source': '/v1/served-models', + 'qualification': 'Served (model, profile) pairs. Inspect the selected model capabilities before admission.'} + + +def _openai_models(raw): + value = strict_json(raw) + entries = value.get('data') if type(value) is dict else None + if type(entries) is not list: + raise ValueError('Invalid bounded Pumas served model listing.') + aliases = {} + for item in entries: + if type(item) is not dict: + raise ValueError('Invalid model listing entry.') + profiles = aliases.setdefault(alias(item.get('id')), []) + # v0.8 lists each ID once with `profiles`; earlier gateways omit it. + listed = item.get('profiles') + for selected in listed if type(listed) is list else (): + if isinstance(selected, str) and re.fullmatch(r'[A-Za-z0-9_.-]{1,128}', selected) and selected not in profiles: + profiles.append(selected) + return {'models': _listing(aliases), 'source': '/v1/models', + 'qualification': 'Serving aliases only. Inspect the selected model capabilities before admission.'} + + def models(base): + """Prefer the typed served-model listing; fall back to `/v1/models` on gateways without it.""" parts = urlsplit(endpoint(base)) - def decode(raw): - value = strict_json(raw) - entries = value.get('data') if type(value) is dict else None - if type(entries) is not list or len(entries) > 512: - raise ValueError('Invalid bounded Pumas served model listing.') - names = [] - for item in entries: - if type(item) is not dict: - raise ValueError('Invalid model listing entry.') - name = alias(item.get('id')) - # PR54 emits one catalog row per served profile and omits profile - # in these rows. Exact-profile capabilities remain admission authority. - if name not in names: - names.append(name) - return {'models': [{'id': name, 'name': name} for name in names], - 'qualification': 'Serving aliases only. Inspect the selected model capabilities before admission.'} + def missing(status, raw): + if status == 404: + raise _RouteMissing() + http_error(status, raw) + try: + return gateway_discovery.get_json(parts.hostname, parts.port, '/v1/served-models', time.monotonic()+3, + max_response=MAX_SERVED_MODELS, decode=_served_models, require_complete=True, error=missing) + except _RouteMissing: + pass return gateway_discovery.get_json(parts.hostname, parts.port, '/v1/models', time.monotonic()+3, - max_response=1024*1024, decode=decode, require_complete=True) - - -def require(observed, name, input_format, output_format, options): - item = next((row for row in observed['capabilities']['capabilities'] if row['capability'] == name), None) - if (item is None or item['availability']['state'] != 'available' or input_format not in item['input_formats'] - or output_format not in item['output_formats']): - raise OperationError('Selected model does not currently advertise the required '+name+' capability.') - bounds = {row['option']: row for row in item['option_bounds']} + max_response=1024*1024, decode=_openai_models, require_complete=True, error=http_error) + + +def descriptor(observed, name): + """Select a capability descriptor by name, never by position or count.""" + return next((row for row in observed['capabilities']['capabilities'] + if type(row) is dict and row.get('capability') == name), None) + + +def require(observed, name, input_format, output_format, options, advisory=()): + """Check live availability and bounds. `advisory` options are checked only when advertised.""" + item = descriptor(observed, name) + if item is None: + raise OperationError('Selected model does not advertise the '+name+' capability.') + availability = item['availability'] + if availability.get('state') != 'available': + reason = availability.get('reason') + raise OperationError('Selected model reports '+name+' unavailable' + + (' ('+message_text(reason)+')' if isinstance(reason, str) else '') + '.') + if input_format not in item['input_formats'] or output_format not in item['output_formats']: + raise OperationError('Selected model does not advertise '+input_format+' to '+output_format+' for '+name+'.') + bounds = {row['option']: row for row in item['option_bounds'] + if type(row) is dict and type(row.get('option')) is str and row['option'] in USED_OPTIONS} for key, value in options.items(): - if key not in bounds or not bounds[key]['minimum'] <= value <= bounds[key]['maximum']: - raise OperationError('Selected model does not advertise the requested '+key+' bound.') + bound = bounds.get(key) + if bound is None: + if key in advisory: + continue + raise OperationError('Selected model does not advertise a '+key+' bound for '+name+'.') + if not bound['minimum'] <= value <= bound['maximum']: + raise OperationError(f"Requested {key} {value} is outside the selected model's {name} bound " + f"{bound['minimum']:g}–{bound['maximum']:g}.") def chat_request(job, user_text, max_tokens): @@ -236,9 +379,8 @@ def watch(): try: value = result(raw, payload['request_id']) except OperationError as error: + # The typed error's `outcome` is authoritative; HTTP status is evidence only. error.provider_http_status = response.status - if error.provider_code and response.status not in (400, 413, 422, 502, 503): - error.outcome = 'unknown' raise if value['kind'] != {'text': 'text', 'png_base64': 'image'}.get(payload['output']): raise OperationError('Typed result has the wrong output kind; no replay.', 'unknown', raw=raw) @@ -259,35 +401,45 @@ def watch(): def result(raw, request_id): + """Decode a finite typed response. Unknown response fields are ignored.""" try: value = strict_json(raw) - object_fields(value, ('contract_version', 'request_id'), ('result', 'error')) - if type(value['contract_version']) is not int or value['contract_version'] != 1 or value['request_id'] != request_id or ('result' in value) == ('error' in value): - raise ValueError('Wrong typed operation correlation.') + if type(value) is not dict or type(value.get('contract_version')) is not int or value['contract_version'] != 1: + raise ValueError('Typed response is not contract version 1.') if 'error' in value: - error = value['error']; object_fields(error, ('code', 'outcome')) - if type(error['code']) is not str or error['code'] not in ERROR_CODES or error['outcome'] not in ('not_admitted', 'unknown'): - raise ValueError('Invalid typed operation error.') - raise OperationError('Pumas '+error['code']+'; outcome '+error['outcome']+'. No automatic replay.', error['outcome'], error['code'], raw) + error = typed_error(value) + if error is None: + detail = message_text(value['error'].get('message')) if type(value['error']) is dict else '' + raise ValueError('Invalid typed operation error' + (': ' + detail if detail else '.')) + # A null request_id means Pumas could not read the body; it is still a typed rejection. + if value.get('request_id') not in (request_id, None): + raise ValueError('Typed error belongs to another request.') + raise OperationError(error_message(error), error['outcome'], error['code'], raw) + response_fields(value, ('request_id', 'result')) + if value['request_id'] != request_id: + raise ValueError('Typed result belongs to another request.') projected = value['result'] if type(projected) is not dict: raise ValueError('Invalid typed result.') if projected.get('kind') == 'text': - object_fields(projected, ('kind', 'text', 'finish_reason')) - if not isinstance(projected['text'], str) or projected['finish_reason'] != 'stop': - raise ValueError('A complete stop-terminated text result is required.') + response_fields(projected, ('kind', 'text', 'finish_reason')) + if not isinstance(projected['text'], str): + raise ValueError('Invalid typed text result.') + if projected['finish_reason'] != 'stop': + raise ValueError('Text did not finish with stop; partial output is not applied.') projected['text'].encode('utf-8') elif projected.get('kind') == 'image': - object_fields(projected, ('kind', 'png_base64', 'seed')) + response_fields(projected, ('kind', 'png_base64', 'seed')) if not isinstance(projected['png_base64'], str) or type(projected['seed']) is not int or not 0 <= projected['seed'] <= 4294967295: raise ValueError('Invalid typed image result.') else: - raise ValueError('Unsupported typed result format.') + raise ValueError('Unsupported typed result kind.') return projected except OperationError: raise - except (ValueError, UnicodeError, TypeError): - raise OperationError('Malformed or incomplete typed result; provider outcome unknown, no replay.', 'unknown', raw=raw) from None + except (ValueError, UnicodeError, TypeError) as error: + raise OperationError('Malformed or incomplete typed result; provider outcome unknown, no replay. '+str(error), + 'unknown', raw=raw) from None def image_dimensions(width, height): diff --git a/pumas_vision.py b/pumas_vision.py index 65ba2f2f..59a677ab 100644 --- a/pumas_vision.py +++ b/pumas_vision.py @@ -1,4 +1,4 @@ -"""Finite Image→Text consumer pinned to combined Pumas PR68, never model qualification. +"""Finite Image→Text consumer of the Pumas v0.8 typed contract, never model qualification. The application sends its prepared image bytes, without URL/path authority. Pumas's production PNG/turbojpeg decoder remains the admission authority. @@ -17,7 +17,7 @@ import pumas_operations as pumas from workbench import encode -SOURCE_COMMIT = '36c2aab2c885b8416fd4e39cf8c37c510596d6e4' +SOURCE_COMMIT = pumas.SOURCE_COMMIT SCHEMA = 'pumas.model-operations.image-to-text' MAX_REQUEST = 32 * 1024 * 1024 MAX_IMAGE = 8 * 1024 * 1024 @@ -26,6 +26,9 @@ MAX_AXIS = 4096 MAX_PIXELS = 4194304 MAX_MESSAGES = MAX_PARTS = 128 +# Pumas has no fixed generation cap; the live max_tokens bound follows the served context. +MAX_TOKENS = 4294967295 +IMAGE_FACTS = ('image_count', 'image_bytes', 'image_pixels', 'width', 'height') def _container(data, encoding): @@ -142,7 +145,8 @@ def validate_input(value, *, facade=False): def request(job, input_value, *, facade=True, max_tokens=512, temperature=None, top_p=None): - if type(max_tokens) is not int or not 1 <= max_tokens <= 2048: raise ValueError('Tokens must be 1–2048.') + if type(max_tokens) is not int or not 1 <= max_tokens <= MAX_TOKENS: + raise ValueError('Tokens must be a positive integer; the selected model bounds the maximum.') options = {'kind': 'text_generation', 'max_tokens': max_tokens} for key, value, maximum in [('temperature', temperature, 2), ('top_p', top_p, 1)]: if value is not None: @@ -171,7 +175,8 @@ def prepare(job, payload, stop): payload['profile'] = observed['capabilities']['profile'] input_format, images = validate_input(payload['input'], facade='semantic_task' in payload) options = {key: value for key, value in payload['options'].items() if key != 'kind'} - pumas.require(observed, 'image_to_text', input_format, 'text', dict(options, image_count=len(images))) - for image in images: pumas.require(observed, 'image_to_text', input_format, 'text', image) + # Image facts are checked against advertised bounds; local preflight already bounds them. + pumas.require(observed, 'image_to_text', input_format, 'text', dict(options, image_count=len(images)), advisory=IMAGE_FACTS) + for image in images: pumas.require(observed, 'image_to_text', input_format, 'text', image, advisory=IMAGE_FACTS) job.update(capability_observation=observed, build_observation=build, producer_contract_source=SOURCE_COMMIT) pumas.request_evidence(job, payload) diff --git a/tests/fake_ai.py b/tests/fake_ai.py index 75e4fb2d..d74e7f34 100644 --- a/tests/fake_ai.py +++ b/tests/fake_ai.py @@ -26,6 +26,12 @@ def do_POST(self): self.send_response(404);self.send_header('Content-Length','0');self.end_headers();return body=json.loads(self.rfile.read(int(self.headers['Content-Length']))) requests.append((self.path,body,dict(self.headers))) + if body['model']=='pumas-conflict': + encoded=json.dumps({'error':{'message':'model corners-test is served by profiles a, b; select one','type':'pumas_error','code':'ambiguous_model'}}).encode() + self.send_response(409);self.send_header('Content-Length',str(len(encoded)));self.end_headers();self.wfile.write(encoded);return + if body['model']=='plain-conflict': + encoded=json.dumps({'error':{'message':'private upstream wording'}}).encode() + self.send_response(409);self.send_header('Content-Length',str(len(encoded)));self.end_headers();self.wfile.write(encoded);return if body['model']=='missing-projector': encoded=json.dumps({'error':{'message':'image input is not supported - provide the mmproj'}}).encode() self.send_response(500);self.send_header('Content-Length',str(len(encoded)));self.end_headers();self.wfile.write(encoded);return diff --git a/tests/fake_images.py b/tests/fake_images.py index 885af67c..f7103fab 100644 --- a/tests/fake_images.py +++ b/tests/fake_images.py @@ -50,6 +50,9 @@ def do_POST(self): return if body['prompt'] == 'busy': return self.reply({'error': {'code': 'runtime_busy'}}, 409) + if body['prompt'] == 'pumas-refusal': + return self.reply({'error': {'message': 'width 33 is not a multiple of 8 for this model; token=sk-abcdefghijkl', + 'type': 'pumas_error', 'code': 'invalid_request'}}, 400) if body['prompt'] == 'unprocessable': return self.reply({'detail': 'fixture contract mismatch'}, 422) raw = b'invalid PNG' if body['prompt'] == 'invalid' else png((body['width'], body['height']), '#467e9a' if body.get('seed', 7) == 7 else '#%06x' % (body['seed'] % 16777216)) diff --git a/tests/fake_pumas_typed.py b/tests/fake_pumas_typed.py index b402ba68..e28f311e 100644 --- a/tests/fake_pumas_typed.py +++ b/tests/fake_pumas_typed.py @@ -24,6 +24,8 @@ def reply(self, value, status=200): try: self.wfile.write(raw) except OSError: pass def do_GET(self): + if self.path == '/v1/served-models' and state.get('served_models') is not None: + return self.reply(state['served_models']) if self.path == '/v1/models': rows = [{'id': 'controlled-text'}, {'id': 'controlled-image'}] if state.get('duplicate_alias'): rows.append({'id': 'controlled-text'}) @@ -31,7 +33,13 @@ def do_GET(self): query = parse_qs(urlsplit(self.path).query) model = query.get('model', [''])[0] if model not in ('controlled-text', 'controlled-image'): + if urlsplit(self.path).path == '/v1/capabilities': + return self.reply({'contract_version': 1, 'request_id': None, 'error': {'code': 'model_not_found', + 'outcome': 'not_admitted', 'message': 'no served model is named ' + repr(model)}}, 404) return self.reply({'error': 'unknown model'}, 404) + if state.get('ambiguous'): + return self.reply({'contract_version': 1, 'request_id': None, 'error': {'code': 'ambiguous_model', + 'outcome': 'not_admitted', 'message': 'alias is served by profiles a, b; send profile'}}, 409) manifest = json.loads((FIXTURES / (model+'-capabilities.json')).read_bytes()) if state['mode'] == 'wrong_profile': manifest['profile'] = 'foreign' if state['mode'] == 'unavailable': @@ -47,7 +55,11 @@ def do_POST(self): if mode == 'loss': self.close_connection = True; return if mode in ('not_admitted', 'unknown'): return self.reply({'contract_version': 1, 'request_id': body['request_id'], - 'error': {'code': 'capability_unavailable' if mode == 'not_admitted' else 'provider_failure', 'outcome': mode}}, 503) + 'error': {'code': 'capability_unavailable' if mode == 'not_admitted' else 'provider_failure', 'outcome': mode, + 'message': state.get('error_message', 'fixture provider reason')}}, state.get('error_status', 503)) + if mode == 'unreadable': + return self.reply({'contract_version': 1, 'request_id': None, 'error': {'code': 'invalid_request', + 'outcome': 'not_admitted', 'message': 'request body is not valid JSON'}}, 400) result = {'kind': 'text', 'text': state['content'], 'finish_reason': 'stop'} if body['capability'] == 'image_generation': import io diff --git a/tests/fake_pumas_vision.py b/tests/fake_pumas_vision.py index 2da5684c..a1eda85e 100644 --- a/tests/fake_pumas_vision.py +++ b/tests/fake_pumas_vision.py @@ -35,7 +35,16 @@ def do_GET(self): value = json.loads((FIXTURES / 'controlled-capabilities.json').read_bytes()) query = parse_qs(urlsplit(self.path).query) if query.get('profile', ['vision-cpu'])[0] != 'vision-cpu': return self.reply({},404) - if state['mode'] == 'unavailable': value['capabilities'][-1]['availability'] = {'state':'unavailable','reason':'runtime_unavailable'} + vision = next(row for row in value['capabilities'] if row['capability'] == 'image_to_text') + if state['mode'] == 'unavailable': vision['availability'] = {'state':'unavailable','reason':'runtime_unavailable'} + if state['mode'] == 'additive': + # v0.8 additive growth: new fields, capabilities, formats, options and reasons. + value['future_field'] = {'nested': True} + value['capabilities'].insert(0, {'capability':'future_capability','semantic_task':'future_task','input_formats':['future'], + 'output_formats':['future'],'streaming':False,'availability':{'state':'deferred'},'option_bounds':[],'extra':1}) + vision['input_formats'].append('future_format'); vision['future'] = 'ignored' + vision['option_bounds'].insert(0, {'option':'future_option','minimum':0.0,'maximum':1.0,'step':1}) + value['capabilities'][-1]['availability'] = {'state':'unavailable','reason':'future_reason'} if state['mode'] == 'wrong_profile': value['profile'] = 'foreign' if state['mode'] == 'wrong_model': value['model'] = 'foreign' return self.reply(value) @@ -47,11 +56,15 @@ def do_POST(self): if mode == 'loss': self.close_connection = True; return if mode in ('not_admitted','unknown'): return self.reply({'contract_version':1,'request_id':body['request_id'], - 'error':{'code':'unsupported_modality' if mode=='not_admitted' else 'invalid_provider_result','outcome':mode}},state['refusal_status']) + 'error':{'code':state.get('error_code') or ('unsupported_modality' if mode=='not_admitted' else 'invalid_provider_result'), + 'outcome':mode,'message':state.get('error_message','fixture reason')}},state['refusal_status']) + if mode == 'unreadable': + return self.reply({'contract_version':1,'request_id':None, + 'error':{'code':'invalid_request','outcome':'not_admitted','message':'request body is not valid JSON'}},400) value = {'contract_version':1,'request_id':body['request_id'], 'result':{'kind':'text','text':state['content'],'finish_reason':'stop'}} if mode == 'wrong_id': value['request_id'] = 'foreign' - if mode == 'private': value['model'] = 'provider/private' + if mode == 'additive': value['future'] = {'x': 1}; value['result']['future'] = 'ignored' if mode == 'length': value['result']['finish_reason'] = 'length' if mode == 'nonfinite': value['extra'] = float('nan') if mode == 'oversized': value['result']['text'] = 'x'*300000 diff --git a/tests/fixtures/pumas-image-to-text-v1/controlled-capabilities.json b/tests/fixtures/pumas-image-to-text-v1/controlled-capabilities.json index 27a4122a..262544db 100644 --- a/tests/fixtures/pumas-image-to-text-v1/controlled-capabilities.json +++ b/tests/fixtures/pumas-image-to-text-v1/controlled-capabilities.json @@ -7,6 +7,7 @@ "max_request_bytes": 33554432, "max_response_bytes": 33554432, "max_stream_event_bytes": 262144, + "context_tokens": 8192, "capabilities": [ { "capability": "chat_generation", @@ -25,7 +26,7 @@ { "option": "max_tokens", "minimum": 1.0, - "maximum": 4294967295.0 + "maximum": 8192.0 }, { "option": "temperature", @@ -56,7 +57,7 @@ { "option": "max_tokens", "minimum": 1.0, - "maximum": 4294967295.0 + "maximum": 8192.0 }, { "option": "temperature", @@ -89,17 +90,7 @@ { "option": "dimensions", "minimum": 1.0, - "maximum": 8192.0 - }, - { - "option": "input_count", - "minimum": 1.0, - "maximum": 128.0 - }, - { - "option": "input_characters", - "minimum": 0.0, - "maximum": 65536.0 + "maximum": 4294967295.0 } ] }, @@ -158,8 +149,8 @@ "option_bounds": [ { "option": "max_output_tokens", - "minimum": 512, - "maximum": 512 + "minimum": 512.0, + "maximum": 512.0 } ] }, @@ -176,7 +167,7 @@ "streaming": false, "availability": { "state": "unavailable", - "reason": "unqualified_audio_runtime" + "reason": "unsupported_adapter" }, "option_bounds": [] }, @@ -198,43 +189,114 @@ "option_bounds": [ { "option": "max_tokens", - "minimum": 1, - "maximum": 2048 + "minimum": 1.0, + "maximum": 8192.0 }, { "option": "temperature", - "minimum": 0, - "maximum": 2 + "minimum": 0.0, + "maximum": 2.0 }, { "option": "top_p", - "minimum": 0, - "maximum": 1 + "minimum": 0.0, + "maximum": 1.0 }, { "option": "image_bytes", - "minimum": 1, - "maximum": 8388608 + "minimum": 1.0, + "maximum": 8388608.0 }, { "option": "image_pixels", - "minimum": 1, - "maximum": 4194304 + "minimum": 1.0, + "maximum": 4194304.0 }, { "option": "image_count", - "minimum": 1, - "maximum": 4 + "minimum": 1.0, + "maximum": 4.0 }, { "option": "width", - "minimum": 1, - "maximum": 4096 + "minimum": 1.0, + "maximum": 4096.0 }, { "option": "height", - "minimum": 1, - "maximum": 4096 + "minimum": 1.0, + "maximum": 4096.0 + } + ] + }, + { + "capability": "multimodal_embedding", + "semantic_task": "multimodal_embedding", + "input_formats": [ + "png_base64", + "jpeg_base64", + "content_batch" + ], + "output_formats": [ + "embeddings_float32" + ], + "streaming": false, + "availability": { + "state": "unavailable", + "reason": "model_task_mismatch" + }, + "option_bounds": [ + { + "option": "dimensions", + "minimum": 1.0, + "maximum": 4294967295.0 + }, + { + "option": "image_bytes", + "minimum": 1.0, + "maximum": 8388608.0 + }, + { + "option": "image_pixels", + "minimum": 1.0, + "maximum": 4194304.0 + }, + { + "option": "image_count", + "minimum": 1.0, + "maximum": 4.0 + } + ] + }, + { + "capability": "decision", + "semantic_task": "decision", + "input_formats": [ + "decision" + ], + "output_formats": [ + "decisions" + ], + "streaming": false, + "availability": { + "state": "unavailable", + "reason": "model_task_mismatch" + }, + "option_bounds": [ + { + "option": "image_bytes", + "minimum": 1.0, + "maximum": 8388608.0 + }, + { + "option": "image_pixels", + "minimum": 1.0, + "maximum": 4194304.0 + }, + { + "option": "image_count", + "minimum": 0.0, + "maximum": 4.0 } ] } diff --git a/tests/fixtures/pumas-image-to-text-v1/schemas/capabilities-response.schema.json b/tests/fixtures/pumas-image-to-text-v1/schemas/capabilities-response.schema.json index d9534ecd..c455d4f7 100644 --- a/tests/fixtures/pumas-image-to-text-v1/schemas/capabilities-response.schema.json +++ b/tests/fixtures/pumas-image-to-text-v1/schemas/capabilities-response.schema.json @@ -43,16 +43,30 @@ "type": "string" }, "Capability": { - "enum": [ - "chat_generation", - "text_generation", - "text_embedding", - "image_generation", - "image_to_text", - "audio_transcription", - "audio_classification" - ], - "type": "string" + "oneOf": [ + { + "enum": [ + "chat_generation", + "text_generation", + "text_embedding", + "image_generation", + "image_to_text", + "audio_transcription", + "audio_classification" + ], + "type": "string" + }, + { + "const": "multimodal_embedding", + "description": "One embedding per image or per mixed text/image item, in the same\nvector space as the model's text embeddings.", + "type": "string" + }, + { + "const": "decision", + "description": "Typed questions about a state answered with probabilities by a native\ndecision model (llama.cpp `/v1/systemone`, e.g. Kev, OpenJev).", + "type": "string" + } + ] }, "CapabilityDescriptor": { "properties": { @@ -107,7 +121,9 @@ "text", "text_batch", "pcm_s16le", - "pcm_f32le" + "pcm_f32le", + "content_batch", + "decision" ], "type": "string" }, @@ -155,7 +171,8 @@ "text", "embeddings_float32", "png_base64", - "labels" + "labels", + "decisions" ], "type": "string" }, @@ -167,7 +184,9 @@ "text_to_image", "speech_to_text", "audio_classification", - "image_to_text" + "image_to_text", + "multimodal_embedding", + "decision" ], "type": "string" } @@ -180,6 +199,15 @@ }, "type": "array" }, + "context_tokens": { + "description": "Context window configured for this serving, when the runtime reports it.", + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, "max_request_bytes": { "format": "uint", "minimum": 0, diff --git a/tests/fixtures/pumas-image-to-text-v1/schemas/capability-descriptor.schema.json b/tests/fixtures/pumas-image-to-text-v1/schemas/capability-descriptor.schema.json index 0377d31b..9dc9d126 100644 --- a/tests/fixtures/pumas-image-to-text-v1/schemas/capability-descriptor.schema.json +++ b/tests/fixtures/pumas-image-to-text-v1/schemas/capability-descriptor.schema.json @@ -43,16 +43,30 @@ "type": "string" }, "Capability": { - "enum": [ - "chat_generation", - "text_generation", - "text_embedding", - "image_generation", - "image_to_text", - "audio_transcription", - "audio_classification" - ], - "type": "string" + "oneOf": [ + { + "enum": [ + "chat_generation", + "text_generation", + "text_embedding", + "image_generation", + "image_to_text", + "audio_transcription", + "audio_classification" + ], + "type": "string" + }, + { + "const": "multimodal_embedding", + "description": "One embedding per image or per mixed text/image item, in the same\nvector space as the model's text embeddings.", + "type": "string" + }, + { + "const": "decision", + "description": "Typed questions about a state answered with probabilities by a native\ndecision model (llama.cpp `/v1/systemone`, e.g. Kev, OpenJev).", + "type": "string" + } + ] }, "InputFormat": { "enum": [ @@ -63,7 +77,9 @@ "text", "text_batch", "pcm_s16le", - "pcm_f32le" + "pcm_f32le", + "content_batch", + "decision" ], "type": "string" }, @@ -111,7 +127,8 @@ "text", "embeddings_float32", "png_base64", - "labels" + "labels", + "decisions" ], "type": "string" }, @@ -123,7 +140,9 @@ "text_to_image", "speech_to_text", "audio_classification", - "image_to_text" + "image_to_text", + "multimodal_embedding", + "decision" ], "type": "string" } diff --git a/tests/fixtures/pumas-image-to-text-v1/schemas/error-response.schema.json b/tests/fixtures/pumas-image-to-text-v1/schemas/error-response.schema.json index 89f00acb..a7694c93 100644 --- a/tests/fixtures/pumas-image-to-text-v1/schemas/error-response.schema.json +++ b/tests/fixtures/pumas-image-to-text-v1/schemas/error-response.schema.json @@ -22,13 +22,18 @@ "code": { "$ref": "#/$defs/ErrorCode" }, + "message": { + "description": "Human-readable reason. Consumers display or log it; they branch on `code`.", + "type": "string" + }, "outcome": { "$ref": "#/$defs/Outcome" } }, "required": [ "code", - "outcome" + "outcome", + "message" ], "type": "object" }, diff --git a/tests/fixtures/pumas-image-to-text-v1/schemas/modality-request.schema.json b/tests/fixtures/pumas-image-to-text-v1/schemas/modality-request.schema.json index dc979861..0f014046 100644 --- a/tests/fixtures/pumas-image-to-text-v1/schemas/modality-request.schema.json +++ b/tests/fixtures/pumas-image-to-text-v1/schemas/modality-request.schema.json @@ -27,6 +27,32 @@ ], "type": "string" }, + "Capability": { + "oneOf": [ + { + "enum": [ + "chat_generation", + "text_generation", + "text_embedding", + "image_generation", + "image_to_text", + "audio_transcription", + "audio_classification" + ], + "type": "string" + }, + { + "const": "multimodal_embedding", + "description": "One embedding per image or per mixed text/image item, in the same\nvector space as the model's text embeddings.", + "type": "string" + }, + { + "const": "decision", + "description": "Typed questions about a state answered with probabilities by a native\ndecision model (llama.cpp `/v1/systemone`, e.g. Kev, OpenJev).", + "type": "string" + } + ] + }, "Content": { "anyOf": [ { @@ -40,6 +66,71 @@ } ] }, + "ContentItem": { + "additionalProperties": false, + "properties": { + "parts": { + "items": { + "$ref": "#/$defs/ImagePart" + }, + "type": "array" + } + }, + "required": [ + "parts" + ], + "type": "object" + }, + "DecisionImage": { + "additionalProperties": false, + "properties": { + "data_base64": { + "type": "string" + }, + "encoding": { + "$ref": "#/$defs/ImageEncoding" + } + }, + "required": [ + "encoding", + "data_base64" + ], + "type": "object" + }, + "DecisionQuestion": { + "additionalProperties": false, + "properties": { + "criteria": true, + "instructions": true, + "type": { + "$ref": "#/$defs/DecisionType" + } + }, + "required": [ + "type", + "instructions" + ], + "type": "object" + }, + "DecisionType": { + "oneOf": [ + { + "const": "choice", + "description": "Pick one option; `criteria` maps option to description (or null).", + "type": "string" + }, + { + "const": "score", + "description": "Rate on a scale; `criteria` lists level descriptions, lowest first.", + "type": "string" + }, + { + "const": "noul", + "description": "Probability that the answer is true; optional `criteria` describes\n`true` and `false`.", + "type": "string" + } + ] + }, "ImageEncoding": { "enum": [ "png", @@ -47,6 +138,67 @@ ], "type": "string" }, + "ImageMessage": { + "additionalProperties": false, + "properties": { + "content": { + "items": { + "$ref": "#/$defs/ImagePart" + }, + "type": "array" + }, + "role": { + "$ref": "#/$defs/Role" + } + }, + "required": [ + "role", + "content" + ], + "type": "object" + }, + "ImagePart": { + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "text", + "type": "string" + }, + "text": { + "type": "string" + } + }, + "required": [ + "kind", + "text" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "data_base64": { + "type": "string" + }, + "encoding": { + "$ref": "#/$defs/ImageEncoding" + }, + "kind": { + "const": "image", + "type": "string" + } + }, + "required": [ + "kind", + "encoding", + "data_base64" + ], + "type": "object" + } + ] + }, "Input": { "oneOf": [ { @@ -106,6 +258,27 @@ ], "type": "object" }, + { + "additionalProperties": false, + "description": "Messages whose content is a list of text and image parts.", + "properties": { + "kind": { + "const": "image_messages", + "type": "string" + }, + "messages": { + "items": { + "$ref": "#/$defs/ImageMessage" + }, + "type": "array" + } + }, + "required": [ + "kind", + "messages" + ], + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -166,6 +339,57 @@ "data_base64" ], "type": "object" + }, + { + "additionalProperties": false, + "description": "Items to embed together; each item produces one vector.", + "properties": { + "items": { + "items": { + "$ref": "#/$defs/ContentItem" + }, + "type": "array" + }, + "kind": { + "const": "content_batch", + "type": "string" + } + }, + "required": [ + "kind", + "items" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "A state and typed questions for a native decision model.", + "properties": { + "images": { + "default": [], + "items": { + "$ref": "#/$defs/DecisionImage" + }, + "type": "array" + }, + "kind": { + "const": "decision", + "type": "string" + }, + "questions": { + "additionalProperties": { + "$ref": "#/$defs/DecisionQuestion" + }, + "type": "object" + }, + "state": true + }, + "required": [ + "kind", + "state", + "questions" + ], + "type": "object" } ] }, @@ -313,6 +537,19 @@ "kind" ], "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "decision", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" } ] }, @@ -323,7 +560,8 @@ "png_base64", "labels", "pcm_s16le", - "pcm_f32le" + "pcm_f32le", + "decisions" ], "type": "string" }, @@ -425,7 +663,9 @@ "text_to_image", "speech_to_text", "audio_classification", - "image_to_text" + "image_to_text", + "multimodal_embedding", + "decision" ], "type": "string" } @@ -433,6 +673,18 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, "properties": { + "capability": { + "anyOf": [ + { + "$ref": "#/$defs/Capability" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional explicit capability. When present, resolution only considers it." + }, "contract_version": { "format": "uint32", "minimum": 0, diff --git a/tests/fixtures/pumas-image-to-text-v1/schemas/operation-request.schema.json b/tests/fixtures/pumas-image-to-text-v1/schemas/operation-request.schema.json index ee426cd0..84219b50 100644 --- a/tests/fixtures/pumas-image-to-text-v1/schemas/operation-request.schema.json +++ b/tests/fixtures/pumas-image-to-text-v1/schemas/operation-request.schema.json @@ -28,16 +28,95 @@ "type": "string" }, "Capability": { - "enum": [ - "chat_generation", - "text_generation", - "text_embedding", - "image_generation", - "image_to_text", - "audio_transcription", - "audio_classification" + "oneOf": [ + { + "enum": [ + "chat_generation", + "text_generation", + "text_embedding", + "image_generation", + "image_to_text", + "audio_transcription", + "audio_classification" + ], + "type": "string" + }, + { + "const": "multimodal_embedding", + "description": "One embedding per image or per mixed text/image item, in the same\nvector space as the model's text embeddings.", + "type": "string" + }, + { + "const": "decision", + "description": "Typed questions about a state answered with probabilities by a native\ndecision model (llama.cpp `/v1/systemone`, e.g. Kev, OpenJev).", + "type": "string" + } + ] + }, + "ContentItem": { + "additionalProperties": false, + "properties": { + "parts": { + "items": { + "$ref": "#/$defs/ImagePart" + }, + "type": "array" + } + }, + "required": [ + "parts" ], - "type": "string" + "type": "object" + }, + "DecisionImage": { + "additionalProperties": false, + "properties": { + "data_base64": { + "type": "string" + }, + "encoding": { + "$ref": "#/$defs/ImageEncoding" + } + }, + "required": [ + "encoding", + "data_base64" + ], + "type": "object" + }, + "DecisionQuestion": { + "additionalProperties": false, + "properties": { + "criteria": true, + "instructions": true, + "type": { + "$ref": "#/$defs/DecisionType" + } + }, + "required": [ + "type", + "instructions" + ], + "type": "object" + }, + "DecisionType": { + "oneOf": [ + { + "const": "choice", + "description": "Pick one option; `criteria` maps option to description (or null).", + "type": "string" + }, + { + "const": "score", + "description": "Rate on a scale; `criteria` lists level descriptions, lowest first.", + "type": "string" + }, + { + "const": "noul", + "description": "Probability that the answer is true; optional `criteria` describes\n`true` and `false`.", + "type": "string" + } + ] }, "ImageEncoding": { "enum": [ @@ -262,6 +341,60 @@ "data_base64" ], "type": "object" + }, + { + "additionalProperties": false, + "description": "Items to embed; each item's text and image parts produce one vector.", + "properties": { + "items": { + "items": { + "$ref": "#/$defs/ContentItem" + }, + "type": "array" + }, + "kind": { + "const": "content_batch", + "type": "string" + } + }, + "required": [ + "kind", + "items" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "A state and typed questions for a decision model.", + "properties": { + "images": { + "description": "Optional images, for decision models with a loaded projector.", + "items": { + "$ref": "#/$defs/DecisionImage" + }, + "type": "array" + }, + "kind": { + "const": "decision", + "type": "string" + }, + "questions": { + "additionalProperties": { + "$ref": "#/$defs/DecisionQuestion" + }, + "description": "Question id to question. Each question is answered independently.", + "type": "object" + }, + "state": { + "description": "Content to evaluate: a string, or JSON given to the model as text." + } + }, + "required": [ + "kind", + "state", + "questions" + ], + "type": "object" } ] }, @@ -393,6 +526,19 @@ "kind" ], "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "decision", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" } ] }, @@ -401,7 +547,8 @@ "text", "embeddings_float32", "png_base64", - "labels" + "labels", + "decisions" ], "type": "string" }, diff --git a/tests/fixtures/pumas-image-to-text-v1/schemas/operation-response.schema.json b/tests/fixtures/pumas-image-to-text-v1/schemas/operation-response.schema.json index 82857c00..277d7a17 100644 --- a/tests/fixtures/pumas-image-to-text-v1/schemas/operation-response.schema.json +++ b/tests/fixtures/pumas-image-to-text-v1/schemas/operation-response.schema.json @@ -1,5 +1,94 @@ { "$defs": { + "DecisionAnswer": { + "oneOf": [ + { + "properties": { + "choice": { + "type": "string" + }, + "confidence": { + "format": "double", + "type": "number" + }, + "probabilities": { + "additionalProperties": { + "format": "double", + "type": "number" + }, + "type": "object" + }, + "type": { + "const": "choice", + "type": "string" + } + }, + "required": [ + "type", + "choice", + "probabilities", + "confidence" + ], + "type": "object" + }, + { + "properties": { + "confidence": { + "format": "double", + "type": "number" + }, + "legend": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "probabilities": { + "additionalProperties": { + "format": "double", + "type": "number" + }, + "type": "object" + }, + "score": { + "description": "Probability-weighted level index; may fall between levels.", + "format": "double", + "type": "number" + }, + "type": { + "const": "score", + "type": "string" + } + }, + "required": [ + "type", + "score", + "legend", + "probabilities", + "confidence" + ], + "type": "object" + }, + { + "properties": { + "noul": { + "description": "Probability that the answer is true.", + "format": "double", + "type": "number" + }, + "type": { + "const": "noul", + "type": "string" + } + }, + "required": [ + "type", + "noul" + ], + "type": "object" + } + ] + }, "FinishReason": { "enum": [ "stop", @@ -74,6 +163,34 @@ "seed" ], "type": "object" + }, + { + "properties": { + "answers": { + "additionalProperties": { + "$ref": "#/$defs/DecisionAnswer" + }, + "type": "object" + }, + "input_tokens": { + "description": "Prompt tokens evaluated across all questions, when reported.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "kind": { + "const": "decisions", + "type": "string" + } + }, + "required": [ + "kind", + "answers" + ], + "type": "object" } ] } diff --git a/tests/fixtures/pumas-image-to-text-v1/source.json b/tests/fixtures/pumas-image-to-text-v1/source.json index e3254144..40d6dc5d 100644 --- a/tests/fixtures/pumas-image-to-text-v1/source.json +++ b/tests/fixtures/pumas-image-to-text-v1/source.json @@ -1,32 +1,45 @@ { "producer_repository": "MrScripty/Pumas-Library", - "producer_pr": 68, - "producer_commit": "36c2aab2c885b8416fd4e39cf8c37c510596d6e4", - "contract_url": "https://github.com/MrScripty/Pumas-Library/blob/36c2aab2c885b8416fd4e39cf8c37c510596d6e4/docs/contracts/image-to-text-v0.8.md", + "producer_branch": "feat/v080-consumer-api", + "producer_commit": "4c79cd8e35db566b70caaa13d8d8f92265f5976e", + "producer_tree": "c646a01d5cc88814bdd2dd7e4565d9d1deda2203", + "contract_path": "docs/contracts/model-operations-v1.md", + "contract_url": "https://github.com/MrScripty/Pumas-Library/blob/4c79cd8e35db566b70caaa13d8d8f92265f5976e/docs/contracts/model-operations-v1.md", + "contract_sha256": "ab0f8a2c9b01f89a28d9a03ed3a9d9246eb6d1665ee2690a4fd2bcb8ff0f9d44", "fixture_kind": "Source-derived controlled consumer fixture; not actual native DTO export or model inference", "inspected_source_sha256": { - "types.rs": "875977948163c4256e60783d540337f117ec70edb1bb3636e7598e516587cca3", - "model_operations.rs": "7c4dfac738db1af29b576691ee929f873b141c7c8e9549e11bc44c4ee1e05c28", - "projection.rs": "444c500edd7ca3ea478d37e68212f8da9325028b87a35f6cdedb3e0e8aaed7a7", - "vision_tests.rs": "e2c175c8d1eab53227d1a91b0fca4621838081ec1d587325ff29312f93be41d6" + "types.rs": "dc4e11a3de059f268c7bbf0f2008b4d1710f5ca6e8c20d06608e67773579bbd0", + "model_operations.rs": "2ffd1b7e215fc0da59fbb70acfbd6a5beb2aa4370fb15546afc7a16f6b49ef16", + "projection.rs": "7357166a8ef9e73b29602323b6ef3d7cca3364b9a18ac0a4715f467e9080f9d8", + "vision_tests.rs": "ce340d0dfe576196330e85b6a92151e6d7329dd9878a0c39e52f0f8126a0dc5a" }, - "schema_export_status": "Six actual production Rust DTO schemas were regenerated at schema_export_commit. The reviewed producer retains identical DTO definitions and schema dependencies; all six SHA-256 identities are unchanged. Handler and controlled-test source hashes are refreshed. The merged main source tree matches the reviewed producer exactly. Packaged/native qualification remains separate.", "inspected_source_native_blob_sha": { - "types.rs": "f7d70f90770deb124dc17dde5212d88b058d9e0e", - "model_operations.rs": "13d15ea4805cfc4df103951d974d76f8987d2964", - "projection.rs": "2a2061a0659a916998455bd22cbc55a7b7f4b0a6", - "vision_tests.rs": "531ff6b48c450800e860a5bc98ea7f00fe27fd3b" + "types.rs": "4b5414db04e62af6d1e39a3c39455ed6bc23e546", + "model_operations.rs": "64ed36cf3606f3028a3a2ec641d02790e71e56fc", + "projection.rs": "c1595de051af74c752e2553e6d2f3c5159c1f47e", + "vision_tests.rs": "4de87a94aba30ec82f19e49c4a8e47ef6de49654" }, - "originating_vision_pr": 65, + "schema_export_status": "Six actual production Rust DTO schemas were regenerated at schema_export_commit with the export-contract feature. The controlled capability fixture follows that commit's descriptor construction for a llama.cpp vision serving with an 8192-token context. Packaged/native qualification remains separate.", "actual_dto_schema_sha256": { - "capabilities-response.schema.json": "b55b150f6958239e71d71f1b28fd36450907ddccf78544d52b64543ad4f2493d", - "capability-descriptor.schema.json": "eb25783123d38de5ff6aedf4cfa69de8f4f5f5c60799768b1b90da258b85f922", - "error-response.schema.json": "b024363b19d55a004d9577ad04d7e2ec9209d3d1193b7c53c46a7d6ed401e3be", - "modality-request.schema.json": "24b5f0a73a1252d31a3b22687a84f07b16a9d3ea700809a42e615e7f4e668704", - "operation-request.schema.json": "08f46f0e79c6cb288c413b008abbdfbedb1898d02fb98d396ef73faf8634c642", - "operation-response.schema.json": "20bf98dfd7292b1e3a99bf34ec524858351ba6f9d226f17257362e9ddf0500f8" + "capabilities-response.schema.json": "e1949bef5e7480f2808f2f1ec66e77adf438e44c6d000a04430bbf15d117a04e", + "capability-descriptor.schema.json": "c586a54fd8c0524dfcbae00dd298d5ee06bb70675c78e871b8a82a77c7ba5316", + "error-response.schema.json": "6b681ab7c7f224e2a35dddaeeb113e9ee2069b058904ced3390c17d1fe36c218", + "modality-request.schema.json": "feb94daa68db1003eb633161ef72431611c1cd5eb6c3096f20c2b69d4a6b9162", + "operation-request.schema.json": "a09fac92af50255d960d1d27ce570c7f89dd377bd7733cead30c4f7fea70a5c5", + "operation-response.schema.json": "17f366d747de6a0047d40a5e549af5abe7045a3a8f64d987c398b7786488d476" }, - "schema_export_command": "PUMAS_IMAGE_CONTRACT_EXPORT_DIR=... ORT_SKIP_DOWNLOAD=1 CARGO_BUILD_JOBS=1 cargo test --locked --offline --manifest-path rust/Cargo.toml -p pumas-rpc --bin pumas-rpc --all-features handlers::model_operations::types::image_to_text_schema_tests::image_to_text_actual_dto_schema_export", - "producer_tree": "cf4d3ca3fbaa77ccd7ba2ebd10c7e9d5ba434ef4", - "schema_export_commit": "5c7d78d7e4eefe65f2395bb3ca2d7bc2ada83c2f" + "schema_export_command": "PUMAS_IMAGE_CONTRACT_EXPORT_DIR=... cargo test --locked -p pumas-rpc --features export-contract image_to_text_actual_dto_schema_export (from rust/)", + "schema_export_commit": "4c79cd8e35db566b70caaa13d8d8f92265f5976e", + "previous_producer": { + "producer_pr": 68, + "producer_commit": "36c2aab2c885b8416fd4e39cf8c37c510596d6e4", + "actual_dto_schema_sha256": { + "capabilities-response.schema.json": "b55b150f6958239e71d71f1b28fd36450907ddccf78544d52b64543ad4f2493d", + "capability-descriptor.schema.json": "eb25783123d38de5ff6aedf4cfa69de8f4f5f5c60799768b1b90da258b85f922", + "error-response.schema.json": "b024363b19d55a004d9577ad04d7e2ec9209d3d1193b7c53c46a7d6ed401e3be", + "modality-request.schema.json": "24b5f0a73a1252d31a3b22687a84f07b16a9d3ea700809a42e615e7f4e668704", + "operation-request.schema.json": "08f46f0e79c6cb288c413b008abbdfbedb1898d02fb98d396ef73faf8634c642", + "operation-response.schema.json": "20bf98dfd7292b1e3a99bf34ec524858351ba6f9d226f17257362e9ddf0500f8" + } + } } diff --git a/tests/test_ai.py b/tests/test_ai.py index dcf601ae..b39c3818 100644 --- a/tests/test_ai.py +++ b/tests/test_ai.py @@ -68,6 +68,13 @@ def test_server_error_explains_missing_projector(self): self.data.suggest(dict(self.body,model='missing-projector')) self.assertIsNone(self.data.sample(self.sample['id'])['annotation']) + def test_pumas_refusal_reason_is_shown_for_any_status(self): + with self.assertRaisesRegex(ai_codex.CodexError, 'HTTP 409.*Pumas reported: model corners-test is served by profiles a, b'): + self.data.suggest(dict(self.body,model='pumas-conflict')) + with self.assertRaises(ai_codex.CodexError) as caught: + self.data.suggest(dict(self.body,model='plain-conflict')) + self.assertNotIn('private upstream wording',str(caught.exception)) + def test_openrouter_catalog_and_auth(self): with patch.object(ai_openrouter,'BASE_URL',self.url): catalog=ai.models(dict(provider='openrouter',api_key='test-key')) diff --git a/tests/test_image_generation.py b/tests/test_image_generation.py index 93e7f74b..576d27e4 100644 --- a/tests/test_image_generation.py +++ b/tests/test_image_generation.py @@ -64,6 +64,23 @@ def test_unprocessable_fields_name_a_contract_mismatch(self): with self.assertRaisesRegex(ValueError, 'contract'): self.manager.generate({**self.body(), 'prompt': 'unprocessable', 'width': 33, 'height': 33}) self.assertEqual(len(self.requests), 1) + def test_pumas_error_message_is_shown_redacted(self): + with self.assertRaises(ValueError) as caught: + self.manager.generate({**self.body(), 'prompt': 'pumas-refusal'}) + message = str(caught.exception) + self.assertIn('Pumas rejected the image request', message) + self.assertIn('Pumas reported: width 33 is not a multiple of 8', message) + self.assertNotIn('sk-abcdefghijkl', message) + with self.assertRaises(ValueError) as caught: + self.manager.generate({**self.body(), 'prompt': 'busy'}) + self.assertNotIn('Pumas reported', str(caught.exception)) + def test_pumas_error_detail_only_reads_pumas_shaped_bodies(self): + import ai_http + self.assertEqual(ai_http.pumas_error_detail(b'{"error":{"message":"m","type":"server_error"}}'), '') + self.assertEqual(ai_http.pumas_error_detail(b''), '') + self.assertEqual(ai_http.pumas_error_detail(b'{"error":{"message":" ","type":"pumas_error"}}'), '') + self.assertEqual(ai_http.pumas_error_detail(b'{"error":{"message":"context window exceeded","type":"pumas_error"}}'), + 'Pumas reported: context window exceeded') def test_no_duration_deadline_remains_and_connect_stays_bounded(self): source = Path(image_generation.__file__).read_text() self.assertFalse(hasattr(image_generation, 'TIMEOUT')) diff --git a/tests/test_pumas_typed.py b/tests/test_pumas_typed.py index b6b76be2..059ddd55 100644 --- a/tests/test_pumas_typed.py +++ b/tests/test_pumas_typed.py @@ -190,18 +190,61 @@ def test_grounded_rewrite_stays_in_source_family_and_draft(self): self.assertEqual(record['review'],'draft'); self.assertEqual(record['parents'],[row['id']]) self.assertEqual(record['groups'],row['groups']); self.assertEqual(record['provenance']['rights'],row['provenance']['rights']) self.assertEqual(record['provenance']['canonical_request']['profile'],'controlled-text-cpu') - def test_manifest_hash_closed_semantics_nonfinite_intervals(self): + def test_manifest_hash_additive_semantics_nonfinite_intervals(self): raw=(FIXTURES/'controlled-text-capabilities.json').read_bytes(); observed=pumas.validate_manifest(raw) self.assertEqual(observed['observed_sha256'],hashlib.sha256(raw).hexdigest()) self.assertEqual(pumas.models(self.url)['models'][0]['id'],'controlled-text') self.state['duplicate_alias']=True self.assertEqual(len(pumas.models(self.url)['models']),2) - for mutate in (lambda v:v.update(extra=1),lambda v:v['capabilities'][0].update(capability={}), - lambda v:v['capabilities'][0]['option_bounds'][0].update(minimum=10**500), + # Additive fields and unrecognised capability entries are ignored, never refused. + for mutate in (lambda v:v.update(extra=1,context_tokens=4096),lambda v:v['capabilities'][0].update(capability={}), + lambda v:v['capabilities'].append({'capability':'decision','semantic_task':'decision'}), + lambda v:v['capabilities'][0]['option_bounds'].append({'option':'top_k','minimum':'n/a'})): + value=copy.deepcopy(observed['capabilities']); mutate(value) + pumas.validate_manifest(json.dumps(value).encode()) + value=copy.deepcopy(observed['capabilities']); value['capabilities'][0].update(capability={}) + with self.assertRaises(pumas.OperationError): + pumas.require(pumas.validate_manifest(json.dumps(value).encode()),'chat_generation','messages_text','text',{'max_tokens':1}) + for mutate in (lambda v:v['capabilities'][0]['option_bounds'][0].update(minimum=10**500), lambda v:v['capabilities'][0]['option_bounds'][0].update(minimum=10,maximum=1), lambda v:v['capabilities'][0].update(availability={'state':'unavailable','reason':{}})): value=copy.deepcopy(observed['capabilities']); mutate(value) with self.assertRaises(ValueError): pumas.validate_manifest(json.dumps(value).encode()) + def test_served_models_listing_profiles_and_fallback(self): + self.state['served_models']={'supported_contract_versions':[1],'max_request_bytes':1,'max_response_bytes':1,'max_stream_event_bytes':1, + 'future':True,'models':[ + {'model':'shared','profile':'gpu','model_id':'lib/a','provider':'llama_cpp','load_state':'loaded','profile_required':True,'capabilities':[],'extra':1}, + {'model':'shared','profile':'cpu','model_id':'lib/a','provider':'llama_cpp','load_state':'loading','profile_required':True,'capabilities':[]}, + {'model':'controlled-text','profile':'controlled-text-cpu','model_id':'lib/t','provider':'ollama','load_state':'loaded','context_tokens':8192, + 'profile_required':False,'capabilities':[]}]} + listed=pumas.models(self.url) + self.assertEqual(listed['source'],'/v1/served-models') + rows={row['id']:row for row in listed['models']} + self.assertEqual(rows['shared']['profiles'],['gpu','cpu']); self.assertTrue(rows['shared']['profile_required']) + self.assertIn('choose one',rows['shared']['name']); self.assertEqual(rows['controlled-text']['name'],'controlled-text') + self.state['served_models']=None + self.assertEqual(pumas.models(self.url)['source'],'/v1/models') + def test_capability_refusals_surface_pumas_message(self): + with self.assertRaises(pumas.OperationError) as caught: pumas.capabilities(self.url,'missing-model') + self.assertIn("no served model is named 'missing-model'",str(caught.exception)); self.assertIn('model_not_found',str(caught.exception)) + self.assertEqual(caught.exception.outcome,'not_admitted') + self.state['ambiguous']=True + with self.assertRaises(pumas.OperationError) as caught: pumas.capabilities(self.url,'controlled-text') + self.assertIn('send profile',str(caught.exception)) + self.state['ambiguous']=False; self.state['mode']='unavailable' + job=self.generate(); self.assertIn('runtime_unavailable',job['error']); self.assertFalse(self.state['requests']) + def test_typed_error_outcome_is_authoritative_and_message_is_retained(self): + for mode,status in (('not_admitted',502),('not_admitted',404),('not_admitted',409),('unknown',400),('unknown',503)): + with self.subTest(mode=mode,status=status): + self.state.update(mode=mode,error_status=status,error_message='provider said '+str(status)) + job=self.generate() + self.assertEqual(job['provider_outcome'],mode); self.assertEqual(job['provider_http_status'],status) + self.assertIn('provider said '+str(status),job['error']) + self.state['mode']='unreadable'; job=self.generate() + self.assertEqual(job['provider_outcome'],'not_admitted'); self.assertIn('not valid JSON',job['error']) + self.state.update(mode='not_admitted',error_status=503,error_message='image runtime is busy') + with self.assertRaises(pumas.OperationError) as caught: self.data.image_requests.generate(self.image()) + self.assertIn('image runtime is busy',str(caught.exception)); self.assertEqual(caught.exception.outcome,'not_admitted') for raw in (b'{"x":1,"x":2}',b'{"x":NaN}',b'{','{"x":1}'.encode('utf-16')): with self.assertRaises(ValueError): pumas.strict_json(raw) diff --git a/tests/test_pumas_vision.py b/tests/test_pumas_vision.py index 22f8a860..91641985 100644 --- a/tests/test_pumas_vision.py +++ b/tests/test_pumas_vision.py @@ -1,4 +1,4 @@ -"""Application acceptance of source-derived vision fixtures and actual DTO schema pins, not inference.""" +"""Application acceptance of source-derived v0.8 vision fixtures and actual DTO schema pins, not inference.""" from auth_helpers import bind_owner import base64 import copy @@ -11,6 +11,7 @@ import uuid from unittest.mock import patch +import jsonschema from PIL import Image from app import Dataset from fake_pumas_vision import FIXTURES, start @@ -39,22 +40,74 @@ def test_actual_dto_schema_identities_and_closed_capability_enum(self): self.assertEqual(hashlib.sha256(raw).hexdigest(), digest, name) self.assertEqual(json.loads(raw)['$schema'], 'https://json-schema.org/draft/2020-12/schema') descriptor = json.loads((FIXTURES/'schemas/capability-descriptor.schema.json').read_text()) - self.assertEqual(set(descriptor['$defs']['Capability']['enum']), set(pumas.CAPABILITIES)) + # Documented capability variants export as `oneOf`; new ones are appended additively. + variants = descriptor['$defs']['Capability']['oneOf'] + names = [name for row in variants for name in row.get('enum', [row.get('const')])] + self.assertTrue(set(pumas.USED_CAPABILITIES) <= set(names)) + self.assertTrue({'multimodal_embedding', 'decision'} <= set(names)) + error = json.loads((FIXTURES/'schemas/error-response.schema.json').read_text()) + self.assertIn('message', error['$defs']['OperationError']['required']) + self.assertEqual(error['properties']['request_id']['type'], ['string', 'null']) for name in ('operation-request', 'modality-request'): schema = json.loads((FIXTURES/'schemas'/(name+'.schema.json')).read_text()) self.assertFalse(schema['additionalProperties']) self.assertEqual(schema['$defs']['ImageEncoding']['enum'], ['png','jpeg']) - def test_seven_source_derived_descriptors_hash_reasons_and_closed_fields(self): + def test_source_derived_descriptors_validate_against_actual_schemas(self): + schemas = {name: json.loads((FIXTURES/'schemas'/(name+'.schema.json')).read_text()) for name in ( + 'capabilities-response', 'operation-request', 'modality-request', 'operation-response', 'error-response')} + def valid(name, value): jsonschema.Draft202012Validator(schemas[name]).validate(value) + valid('capabilities-response', json.loads((FIXTURES/'controlled-capabilities.json').read_bytes())) + valid('operation-request', vision.request(self.job(), image(), facade=False)) + valid('operation-request', vision.request(self.job(), {'kind':'image_messages','messages':[{'role':'user','content':[image()]}]}, facade=False)) + valid('modality-request', {key: value for key, value in vision.request(self.job(), image('jpeg')).items()}) + valid('operation-response', {'contract_version':1,'request_id':'caption-17','result':{'kind':'text','text':'x','finish_reason':'stop'}}) + valid('error-response', {'contract_version':1,'request_id':None,'error':{'code':'invalid_request','outcome':'not_admitted','message':'unreadable body'}}) + def test_source_derived_descriptors_hash_and_select_by_name(self): raw = (FIXTURES/'controlled-capabilities.json').read_bytes() observed = pumas.validate_manifest(raw) self.assertEqual(observed['observed_sha256'],hashlib.sha256(raw).hexdigest()) - self.assertEqual(set(row['capability']for row in observed['capabilities']['capabilities']),set(pumas.CAPABILITIES)) - self.assertEqual(len(pumas.CAPABILITIES),7) - self.assertEqual(observed['capabilities']['capabilities'][4]['option_bounds'][0]['option'],'max_output_tokens') - for mutate in (lambda row:row.update(capability='image_understanding'),lambda row:row['input_formats'].append('image_url'), - lambda row:row['option_bounds'][0].update(maximum=float('inf')),lambda row:row.update(semantic_task='chat_generation')): - value = copy.deepcopy(observed['capabilities']); mutate(value['capabilities'][-1]) - with self.assertRaises(ValueError): pumas.validate_manifest(json.dumps(value).encode()) + names = [row['capability'] for row in observed['capabilities']['capabilities']] + self.assertTrue({'multimodal_embedding','decision'} | set(pumas.USED_CAPABILITIES) <= set(names)) + self.assertEqual(observed['capabilities']['context_tokens'], 8192) + self.assertEqual(pumas.descriptor(observed,'audio_transcription')['option_bounds'][0]['option'],'max_output_tokens') + vision_row = pumas.descriptor(observed,'image_to_text') + self.assertEqual(next(b for b in vision_row['option_bounds'] if b['option']=='max_tokens')['maximum'], 8192) + def test_additive_fields_capabilities_formats_options_and_reasons_are_ignored(self): + value = json.loads((FIXTURES/'controlled-capabilities.json').read_bytes()) + value.update(future_field={'nested':True}, supported_contract_versions=[2,1]) + value['capabilities'].reverse() + value['capabilities'].insert(3, {'capability':'image_understanding','semantic_task':'x','input_formats':[7],'availability':'odd'}) + value['capabilities'].append('not even an object') + row = next(r for r in value['capabilities'] if type(r) is dict and r['capability']=='image_to_text') + row['input_formats'].append('image_url'); row['output_formats'].append('future'); row['future'] = 1 + row['option_bounds'].insert(0, {'option':'future_option','minimum':'unbounded'}) + row['option_bounds'].append({'option':'max_tokens_hint','minimum':1,'maximum':1}) + for item in value['capabilities']: + if type(item) is dict and item.get('capability') == 'decision': + item['availability'] = {'state':'unavailable','reason':'future_reason'} + observed = pumas.validate_manifest(json.dumps(value).encode()) + pumas.require(observed,'image_to_text','messages_image','text',{'max_tokens':4096,'image_count':1}) + with self.assertRaises(pumas.OperationError) as caught: + pumas.require(observed,'image_to_text','messages_image','text',{'max_tokens':8193}) + self.assertIn('8192', str(caught.exception)) + with self.assertRaises(pumas.OperationError) as caught: + pumas.require(observed,'decision','decision','decisions',{}) + self.assertIn('future_reason', str(caught.exception)) + del value['context_tokens']; del value['max_stream_event_bytes'] + pumas.validate_manifest(json.dumps(value).encode()) + def test_used_descriptor_fields_are_still_validated(self): + observed = pumas.validate_manifest((FIXTURES/'controlled-capabilities.json').read_bytes()) + def vision_row(value): return next(r for r in value['capabilities'] if r['capability']=='image_to_text') + for mutate in (lambda v:vision_row(v)['option_bounds'][0].update(maximum=float('inf')), + lambda v:vision_row(v).update(semantic_task='chat_generation'), + lambda v:vision_row(v).update(input_formats='messages_image'), + lambda v:vision_row(v).update(availability={'state':'unavailable','reason':{}}), + lambda v:vision_row(v)['option_bounds'].append({'option':'max_tokens','minimum':1,'maximum':2}), + lambda v:v['capabilities'].append(copy.deepcopy(vision_row(v))), + lambda v:v.update(supported_contract_versions=[2]), + lambda v:v.update(profile=None), lambda v:v.pop('capabilities')): + value = copy.deepcopy(observed['capabilities']); mutate(value) + with self.subTest(value=value), self.assertRaises(ValueError): pumas.validate_manifest(json.dumps(value).encode()) def test_named_and_facade_image_order_and_original_compressed_bytes(self): for encoding in ('png','jpeg'): part = image(encoding,progressive=encoding=='jpeg') @@ -98,18 +151,40 @@ def messages(parts,role='user'):return {'kind':'messages','messages':[{'role':ro with self.assertRaises(ValueError): vision.request(self.job(),value) self.assertEqual((vision.MAX_IMAGE,vision.MAX_AGGREGATE,vision.MAX_REQUEST,vision.MAX_IMAGES),(8388608,16777216,33554432,4)) def test_finite_options_and_correlated_typed_result_without_model(self): - for key,value in [('max_tokens',0),('max_tokens',2049),('max_tokens',True),('temperature',float('nan')), + for key,value in [('max_tokens',0),('max_tokens',2**32),('max_tokens',True),('temperature',float('nan')), ('temperature',3),('top_p',float('inf')),('top_p',-1)]: with self.assertRaises(ValueError): vision.request(self.job(),image(),**{key:value}) - request=vision.request(self.job(),image(),max_tokens=2048,temperature=2,top_p=1) - self.assertEqual(request['options'],{'kind':'text_generation','max_tokens':2048,'temperature':2,'top_p':1}) + # No fixed 2048 cap: the live max_tokens bound follows the served context. + request=vision.request(self.job(),image(),max_tokens=8192,temperature=2,top_p=1) + self.assertEqual(request['options'],{'kind':'text_generation','max_tokens':8192,'temperature':2,'top_p':1}) response={'contract_version':1,'request_id':'caption-17','result':{'kind':'text','text':'A blue rectangle.','finish_reason':'stop'}} self.assertEqual(pumas.result(encode(response).encode(),'caption-17')['text'],'A blue rectangle.') - for mutate in (lambda v:v.update(request_id='foreign'),lambda v:v.update(model='private'), - lambda v:v['result'].update(finish_reason='length'),lambda v:v['result'].update(extra='private')): + for mutate in (lambda v:v.update(model='additive'),lambda v:v['result'].update(extra='additive')): + value=copy.deepcopy(response);mutate(value) + self.assertEqual(pumas.result(encode(value).encode(),'caption-17')['text'],'A blue rectangle.') + for mutate in (lambda v:v.update(request_id='foreign'),lambda v:v['result'].update(finish_reason='length'), + lambda v:v.update(contract_version=2),lambda v:v['result'].update(kind='decisions')): value=copy.deepcopy(response);mutate(value) with self.assertRaises(pumas.OperationError)as caught:pumas.result(encode(value).encode(),'caption-17') self.assertEqual(caught.exception.outcome,'unknown') + def test_typed_errors_use_outcome_message_and_null_request_id(self): + def error(request_id='caption-17', **fields): + body={'code':'model_not_found','outcome':'not_admitted','message':'no served model is named "x"'}; body.update(fields) + return encode({'contract_version':1,'request_id':request_id,'error':body}).encode() + cases=[(error(),'not_admitted','model_not_found'),(error(None,code='invalid_request'),'not_admitted','invalid_request'), + (error(code='ambiguous_operation'),'not_admitted','ambiguous_operation'), + (error(code='future_code'),'not_admitted','future_code'), + (error(code='provider_failure',outcome='unknown'),'unknown','provider_failure'), + (error(outcome='future_outcome'),'unknown','model_not_found')] + for raw,outcome,code in cases: + with self.subTest(raw=raw),self.assertRaises(pumas.OperationError)as caught:pumas.result(raw,'caption-17') + self.assertEqual((caught.exception.outcome,caught.exception.provider_code),(outcome,code)) + self.assertIn('no served model is named "x"',str(caught.exception)) + for raw in (error('foreign'),error(code=7),encode({'contract_version':1,'request_id':'caption-17','error':'text'}).encode()): + with self.subTest(raw=raw),self.assertRaises(pumas.OperationError)as caught:pumas.result(raw,'caption-17') + self.assertEqual(caught.exception.outcome,'unknown') + with self.assertRaises(pumas.OperationError)as caught:pumas.result(error(message='x'*5000+'\nnext'),'caption-17') + self.assertLess(len(str(caught.exception)),800);self.assertNotIn('\n',str(caught.exception)) class VisionApplicationTests(unittest.TestCase): @@ -151,16 +226,24 @@ def test_build_selection_and_unavailability_refuse_without_post(self): for mode in ('missing_build','unavailable','wrong_profile','wrong_model'): self.state['mode']=mode;job=self.generate() self.assertEqual(job['status'],'failed');self.assertEqual(job['provider_outcome'],'not_admitted') + if mode=='unavailable':self.assertIn('runtime_unavailable',job['error']) self.assertFalse(self.state['requests']) with self.assertRaises(ValueError):self.data.caption_proposals.start(self.body(seed=42)) def test_http_refusals_outcomes_and_uncertain_failures_are_never_replayed(self): - for status in (400,413,422,503,502): - mode='unknown'if status==502 else'not_admitted' - self.state.update(mode=mode,refusal_status=status);body=self.body();job=self.generate(body) - self.assertEqual(job['provider_outcome'],mode);self.assertEqual(job['provider_http_status'],status) + # `outcome` is authoritative, not the HTTP status (404/409 are ordinary refusals). + cases=[(400,'not_admitted'),(404,'not_admitted'),(409,'not_admitted'),(413,'not_admitted'),(422,'not_admitted'), + (503,'not_admitted'),(502,'not_admitted'),(502,'unknown'),(500,'unknown'),(400,'unknown')] + for status,mode in cases: + self.state.update(mode=mode,refusal_status=status,error_message='reason for '+str(status));body=self.body();job=self.generate(body) + self.assertEqual(job['provider_outcome'],mode,status);self.assertEqual(job['provider_http_status'],status) + self.assertIn('reason for '+str(status),job['error']) self.data.caption_proposals.start(body) - self.assertEqual(len(self.state['requests']),5) - for mode in ('wrong_id','private','length','nonfinite','oversized','truncated','loss'): + self.state['mode']='unreadable';job=self.generate() + self.assertEqual(job['provider_outcome'],'not_admitted');self.assertEqual(job['provider_error_code'],'invalid_request') + self.assertIn('request body is not valid JSON',job['error']) + self.assertEqual(len(self.state['requests']),len(cases)+1) + self.state['mode']='additive';self.assertEqual(self.generate()['status'],'completed') + for mode in ('wrong_id','length','nonfinite','oversized','truncated','loss'): self.state['mode']=mode;body=self.body();before=len(self.state['requests']);job=self.generate(body) self.assertEqual(job['status'],'failed',mode);self.assertEqual(job['provider_outcome'],'unknown',mode) self.data.caption_proposals.start(body);self.assertEqual(len(self.state['requests']),before+1) From 50bbc884310122d71d0d98e6b9ab6371e0636c58 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 2 Nov 2026 19:25:42 -0700 Subject: [PATCH 2/3] fix(pumas): ignore additive fields in the /.well-known/pumas advertisement The gateway advertisement decoder was closed: any field Pumas added to the advertisement, or more than 64 entries in an open list, made the gateway disappear from scans. Required fields are now checked for presence and type only. Unknown fields are ignored and dropped from the decoded advertisement, so the stored build observation keeps only the known fields; list-length caps give way to the 64 KiB body bound. Missing required fields, wrong types, unsupported schema versions and malformed endpoints still fail, now as descriptor.Rejected with a reason naming the field. A gateway scan returns refused advertisements under `rejected` with their reasons, and the chooser shows them. Co-Authored-By: Claude Opus 5.5 --- .../contracts/pumas-image-to-text-consumer.md | 9 +- gateway_discovery.py | 13 +- pumas_gateway_descriptor.py | 166 +++++++++++------- static/pumas-gateways.js | 4 +- tests/test_pumas_gateways.py | 67 ++++++- 5 files changed, 185 insertions(+), 74 deletions(-) diff --git a/docs/contracts/pumas-image-to-text-consumer.md b/docs/contracts/pumas-image-to-text-consumer.md index 287693cf..c557e28e 100644 --- a/docs/contracts/pumas-image-to-text-consumer.md +++ b/docs/contracts/pumas-image-to-text-consumer.md @@ -40,8 +40,13 @@ Choose **Pumas typed v1** in the image caption form, select the serving alias an optional exact profile, and inspect capabilities. On each new attempt, the owner requires `pumas.model-operations.image-to-text:1` in the gateway build advertisement and an available Image→Text descriptor for the exact resolved model/profile. The -gateway advertisement keeps its own closed, versioned decoder: it is retained as -evidence, so unknown advertisement fields are refused rather than stored. +gateway advertisement (`GET /.well-known/pumas`) has its own versioned decoder, +which also grows additively: unknown fields and new protocol, schema, capability or +compiled-feature entries are ignored and are not retained in the stored build +observation, which keeps only the known fields. Missing required fields, wrong +types, unsupported schema versions and malformed endpoints are refused with a +reason naming the field; a gateway scan lists such refused advertisements with +their reasons instead of hiding them. The request uses the facade's `messages`, `semantic_task:"image_to_text"`, text-generation options and `stream:false`. The seed is omitted. Named image and `image_messages` inputs are also supported by the bounded consumer request builder. diff --git a/gateway_discovery.py b/gateway_discovery.py index db3afc8b..7528c658 100644 --- a/gateway_discovery.py +++ b/gateway_discovery.py @@ -218,15 +218,20 @@ def scan_advertised(): def observe(item): host, port = item address = '[' + host + ']' if ':' in host else host + url = f'http://{address}:{port}' try: - return inspect_advertised(f'http://{address}:{port}', deadline) + return inspect_advertised(url, deadline), None + except descriptor.Rejected as error: + # Pumas answered but its advertisement was refused: keep the reason for the user. + return None, {'server_url': url, 'reason': str(error)} except (OSError, ValueError, TypeError, http.client.HTTPException): - return None + return None, None with ThreadPoolExecutor(max_workers=32) as pool: results = list(pool.map(observe, endpoints)) # Distinct IPv4/IPv6 services on one port must retain their library context. - found = {row['server_url']: row for row in results if row is not None} - return {'gateways': list(found.values()), 'qualification': + found = {row['server_url']: row for row, _ in results if row is not None} + rejected = [refusal for _, refusal in results if refusal is not None] + return {'gateways': list(found.values()), 'rejected': rejected, 'qualification': 'Read-only HTTP advertisements. Core ownership is not authenticated; compiled features do not prove model readiness.'} finally: SCAN_LOCK.release() diff --git a/pumas_gateway_descriptor.py b/pumas_gateway_descriptor.py index eb25b9f1..b1eb3ec0 100644 --- a/pumas_gateway_descriptor.py +++ b/pumas_gateway_descriptor.py @@ -1,12 +1,23 @@ -"""Decode the public Pumas PR51 HTTP advertisement; no owner authentication.""" +"""Decode the public Pumas HTTP advertisement (`GET /.well-known/pumas`); no owner authentication. + +The advertisement grows additively. Unknown fields and new entries in open lists +(protocols, schemas, capabilities, compiled features) are ignored: they are neither +errors nor retained in the decoded advertisement. Missing required fields, wrong +types, unsupported schema versions and malformed endpoints are refused with a +reason naming the field. +""" import hashlib import ipaddress import json from urllib.parse import urlsplit SOURCE_COMMIT = '80f06ab17f9eea639fee143c86319ca9b5e1a21c' +# Transport bound. Open lists have no entry-count cap of their own. MAX_DESCRIPTION_BYTES = 64 * 1024 -MAX_ITEMS = 64 + + +class Rejected(ValueError): + """A Pumas advertisement that was read and refused. The message is the reason.""" def endpoint(value): @@ -28,60 +39,82 @@ def endpoint(value): return f'http://{address}:{port}' -def _text(value, limit=4096): +def _fail(where, problem): + raise Rejected(' '.join(filter(None, ('Pumas advertisement', where, problem))) + '.') + + +def _text(value, where, limit=4096): if not isinstance(value, str) or not value or len(value) > limit: - raise ValueError('Invalid Pumas descriptor text.') - value.encode('utf-8', errors='strict') + _fail(where, f'must be a nonempty string of at most {limit} characters') + try: + value.encode('utf-8', errors='strict') + except UnicodeError: + _fail(where, 'must be valid Unicode text') -def _uint(value): +def _uint(value, where): if type(value) is not int or not 0 <= value <= 4294967295: - raise ValueError('Invalid Pumas descriptor integer.') + _fail(where, 'must be an unsigned 32-bit integer') -def _object(value, required, optional=()): - if type(value) is not dict or not set(required) <= value.keys() or value.keys() - set(required) - set(optional): - raise ValueError('Unsupported Pumas descriptor fields.') +def _version(value, where): + if type(value) is not int: + _fail(where, 'must be an integer') + if value != 1: + raise Rejected(f'Pumas advertisement {where} {value} is unsupported; Tuldok supports version 1.') -def _list(value): - if type(value) is not list or len(value) > MAX_ITEMS: - raise ValueError('Invalid Pumas descriptor list.') +def _object(value, where, required, optional=()): + """Open shape: every required field present. Returns only the known fields.""" + if type(value) is not dict: + _fail(where, 'must be an object') + missing = [name for name in required if name not in value] + if missing: + _fail(where, 'is missing required field(s) ' + ', '.join(missing)) + known = set(required) | set(optional) + return {name: item for name, item in value.items() if name in known} -def _strings(value): - _list(value) - for item in value: - _text(item, 512) +def _list(value, where): + if type(value) is not list: + _fail(where, 'must be a list') + return value -def _protocols(value): - _list(value) - for item in value: - _object(item, ('name', 'versions')) - _text(item['name'], 512) - _list(item['versions']) - for version in item['versions']: - _uint(version) +def _strings(value, where): + for index, item in enumerate(_list(value, where)): + _text(item, f'{where}[{index}]', 512) + return list(value) -def _build(value): - _object(value, ('build_info_schema_version', 'component', 'package_version', 'compiled_features', 'protocols', 'schemas'), - ('build_id', 'source_revision', 'target')) - if type(value['build_info_schema_version']) is not int or value['build_info_schema_version'] != 1: - raise ValueError('Unsupported Pumas build-info schema.') +def _protocols(value, where): + result = [] + for index, item in enumerate(_list(value, where)): + item = _object(item, f'{where}[{index}]', ('name', 'versions')) + _text(item['name'], f'{where}[{index}].name', 512) + for position, version in enumerate(_list(item['versions'], f'{where}[{index}].versions')): + _uint(version, f'{where}[{index}].versions[{position}]') + result.append(item) + return result + + +def _build(value, where): + value = _object(value, where, ('build_info_schema_version', 'component', 'package_version', 'compiled_features', + 'protocols', 'schemas'), ('build_id', 'source_revision', 'target')) + _version(value['build_info_schema_version'], where + '.build_info_schema_version') for name in ('component', 'package_version', 'build_id', 'source_revision', 'target'): - if name in value and value[name] is not None: - _text(value[name]) - elif name in ('component', 'package_version'): - raise ValueError('Missing Pumas build identity.') - _strings(value['compiled_features']) - _protocols(value['protocols']) - _list(value['schemas']) - for item in value['schemas']: - _object(item, ('name', 'version')) - _text(item['name'], 512) - _uint(item['version']) + if name in ('component', 'package_version') or value.get(name) is not None: + _text(value[name], f'{where}.{name}') + value['compiled_features'] = _strings(value['compiled_features'], where + '.compiled_features') + value['protocols'] = _protocols(value['protocols'], where + '.protocols') + schemas = [] + for index, item in enumerate(_list(value['schemas'], where + '.schemas')): + item = _object(item, f'{where}.schemas[{index}]', ('name', 'version')) + _text(item['name'], f'{where}.schemas[{index}].name', 512) + _uint(item['version'], f'{where}.schemas[{index}].version') + schemas.append(item) + value['schemas'] = schemas + return value def _pairs(pairs): @@ -102,29 +135,34 @@ def decode(raw): raise ValueError('Pumas descriptor exceeds the 64 KiB observation limit.') try: value = json.loads(raw.decode('utf-8'), object_pairs_hook=_pairs, parse_constant=_constant) - _object(value, ('advertisement_schema_version', 'service_generation', 'instance', 'endpoint', 'build_info')) - if type(value['advertisement_schema_version']) is not int or value['advertisement_schema_version'] != 1: - raise ValueError('Unsupported Pumas HTTP advertisement schema.') - _text(value['service_generation'], 128) - endpoint(value['endpoint']) - instance = value['instance'] - _object(instance, ('discovery_schema_version', 'registry_library_id', 'library_root', 'generation', 'pumas_version', - 'protocols', 'capabilities', 'model_ref_schema_version', 'selector_schema_version'), ('build_info',)) - if type(instance['discovery_schema_version']) is not int or instance['discovery_schema_version'] != 1: - raise ValueError('Unsupported Pumas discovery schema.') - for name in ('registry_library_id', 'library_root', 'generation', 'pumas_version'): - _text(instance[name]) - for name in ('model_ref_schema_version', 'selector_schema_version'): - _uint(instance[name]) - _protocols(instance['protocols']) - _strings(instance['capabilities']) - if instance.get('build_info') is not None: - _build(instance['build_info']) - build = value['build_info'] - _build(build) - if (not any(p['name'] == 'pumas.local-http' and 1 in p['versions'] for p in build['protocols']) - or not any(s['name'] == 'pumas.http-advertisement' and s['version'] == 1 for s in build['schemas'])): - raise ValueError('Unsupported Pumas HTTP protocol/schema advertisement.') except (UnicodeError, RecursionError) as error: raise ValueError('Invalid Pumas descriptor encoding or nesting.') from error + if type(value) is not dict or 'advertisement_schema_version' not in value: + raise ValueError('Not a Pumas HTTP advertisement: advertisement_schema_version is missing.') + value = _object(value, '', ('advertisement_schema_version', 'service_generation', 'instance', 'endpoint', + 'build_info')) + _version(value['advertisement_schema_version'], 'advertisement_schema_version') + _text(value['service_generation'], 'service_generation', 128) + try: + endpoint(value['endpoint']) + except ValueError as error: + raise Rejected(f'Pumas advertisement endpoint is invalid: {error}') from None + instance = value['instance'] = _object( + value['instance'], 'instance', ('discovery_schema_version', 'registry_library_id', 'library_root', 'generation', + 'pumas_version', 'protocols', 'capabilities', 'model_ref_schema_version', + 'selector_schema_version'), ('build_info',)) + _version(instance['discovery_schema_version'], 'instance.discovery_schema_version') + for name in ('registry_library_id', 'library_root', 'generation', 'pumas_version'): + _text(instance[name], 'instance.' + name) + for name in ('model_ref_schema_version', 'selector_schema_version'): + _uint(instance[name], 'instance.' + name) + instance['protocols'] = _protocols(instance['protocols'], 'instance.protocols') + instance['capabilities'] = _strings(instance['capabilities'], 'instance.capabilities') + if instance.get('build_info') is not None: + instance['build_info'] = _build(instance['build_info'], 'instance.build_info') + build = value['build_info'] = _build(value['build_info'], 'build_info') + if not any(p['name'] == 'pumas.local-http' and 1 in p['versions'] for p in build['protocols']): + raise Rejected('Pumas advertisement build_info does not advertise protocol pumas.local-http version 1.') + if not any(s['name'] == 'pumas.http-advertisement' and s['version'] == 1 for s in build['schemas']): + raise Rejected('Pumas advertisement build_info does not advertise schema pumas.http-advertisement version 1.') return {'advertisement': value, 'observed_sha256': hashlib.sha256(raw).hexdigest()} diff --git a/static/pumas-gateways.js b/static/pumas-gateways.js index f6bc193c..6214e738 100644 --- a/static/pumas-gateways.js +++ b/static/pumas-gateways.js @@ -31,7 +31,9 @@ $('pumas-gateway-scan').addEventListener('click',async()=>{ pumasGateways=result.gateways; $('pumas-gateway-choice').replaceChildren(new Option('Choose an advertised gateway',''),...pumasGateways.map(row=> new Option(row.server_url+' · '+row.advertisement.build_info.package_version+' · '+row.advertisement.instance.registry_library_id,row.server_url))); - pumasGatewayStatus(pumasGateways.length?'Choose a gateway and annotation workflow. Model support is checked when you request a proposal.':'No compatible HTTP advertisements found. Start Pumas with HTTP discovery, or enter a legacy gateway URL in the proposal form.'); + // Advertisements Pumas served but Tuldok refused keep their reasons. + const refused=(Array.isArray(result.rejected)?result.rejected:[]).map(row=>' Refused '+row.server_url+': '+row.reason).join(''); + pumasGatewayStatus((pumasGateways.length?'Choose a gateway and annotation workflow. Model support is checked when you request a proposal.':'No compatible HTTP advertisements found. Start Pumas with HTTP discovery, or enter a legacy gateway URL in the proposal form.')+refused); }catch(error){if(token.epoch===pumasGatewayEpoch&&!pumasGatewayPaused)pumasGatewayStatus(error.message);} finally{if(pumasGatewayOperation===token){pumasGatewayOperation=null;renderPumasGateway();}} }); diff --git a/tests/test_pumas_gateways.py b/tests/test_pumas_gateways.py index 2bda674b..7f7d8005 100644 --- a/tests/test_pumas_gateways.py +++ b/tests/test_pumas_gateways.py @@ -57,7 +57,68 @@ def test_exact_named_identity_optional_provenance_and_raw_hash(self): legacy['instance']['build_info'] = None self.assertIsNone(descriptor.decode(json.dumps(legacy).encode())['advertisement']['instance']['build_info']) - def test_versions_boolean_wrong_shapes_unknown_fields_and_oversizing_fail(self): + def test_additive_fields_and_open_list_entries_are_accepted_and_not_retained(self): + additions = [ + lambda v: v.update(future_field={'nested': [1, 2]}), + lambda v: v['instance'].update(connection_token='never retained'), + lambda v: v['instance']['build_info'].update(future_field=True), + lambda v: v['build_info'].update(future_field='ignored'), + lambda v: v['build_info']['protocols'][0].update(future_field='ignored'), + lambda v: v['build_info']['schemas'][0].update(future_field='ignored'), + lambda v: v['build_info']['schemas'].append({'name': 'pumas.discovery', 'version': 2}), + lambda v: v['build_info']['schemas'].append({'name': 'pumas.future-schema', 'version': 1}), + lambda v: v['build_info']['protocols'].append({'name': 'pumas.future-protocol', 'versions': [3]}), + lambda v: v['build_info']['protocols'][-1].update(versions=[1, 2]), + lambda v: v['instance'].update(capabilities=[f'future.capability{i}@1' for i in range(100)]), + ] + for add in additions: + value = fixture(); add(value) + with self.subTest(value=value): + decoded = descriptor.decode(json.dumps(value).encode())['advertisement'] + self.assertEqual(decoded['instance']['registry_library_id'], fixture()['instance']['registry_library_id']) + text = json.dumps(decoded) + self.assertNotIn('future_field', text) + self.assertNotIn('connection_token', text) + value = fixture(); value['future_field'] = 'ignored'; value['instance']['future_field'] = 'ignored' + self.assertEqual(descriptor.decode(json.dumps(value).encode())['advertisement'], fixture()) + endpoint, calls = self.server(value) + self.assertEqual(discovery.inspect_advertised(f'http://127.0.0.1:{endpoint[1]}')['advertisement']['instance'], + fixture()['instance']) + + def test_missing_required_fields_and_unsupported_versions_are_refused_with_reasons(self): + cases = [ + (lambda v: v.pop('service_generation'), + 'Pumas advertisement is missing required field(s) service_generation.'), + (lambda v: v['instance'].pop('generation'), + 'Pumas advertisement instance is missing required field(s) generation.'), + (lambda v: v['build_info'].pop('schemas'), + 'Pumas advertisement build_info is missing required field(s) schemas.'), + (lambda v: v['build_info']['protocols'][0].pop('versions'), + 'Pumas advertisement build_info.protocols[0] is missing required field(s) versions.'), + (lambda v: v.update(advertisement_schema_version=2), + 'Pumas advertisement advertisement_schema_version 2 is unsupported; Tuldok supports version 1.'), + (lambda v: v['instance'].update(selector_schema_version='1'), + 'Pumas advertisement instance.selector_schema_version must be an unsigned 32-bit integer.'), + (lambda v: v.update(endpoint='http://example.com:8765'), + 'Pumas advertisement endpoint is invalid: Enter a numeric-loopback HTTP base URL.'), + ] + for mutate, reason in cases: + value = fixture(); mutate(value) + with self.subTest(reason=reason): + with self.assertRaises(descriptor.Rejected) as caught: + descriptor.decode(json.dumps(value).encode()) + self.assertEqual(str(caught.exception), reason) + value = fixture(); value['instance'].pop('generation') + endpoint, calls = self.server(value) + with patch.object(discovery, 'listening_endpoints', return_value=[endpoint]): + result = discovery.scan_advertised() + self.assertEqual(result['gateways'], []) + self.assertEqual(result['rejected'], [{'server_url': f'http://127.0.0.1:{endpoint[1]}', + 'reason': 'Pumas advertisement instance is missing required field(s) generation.'}]) + with self.assertRaisesRegex(ValueError, 'missing required field'): + discovery.inspect_advertised(f'http://127.0.0.1:{endpoint[1]}') + + def test_versions_boolean_wrong_shapes_and_oversizing_fail(self): mutations = [ lambda v: v.update(advertisement_schema_version=True), lambda v: v.update(advertisement_schema_version=2), @@ -69,8 +130,8 @@ def test_versions_boolean_wrong_shapes_unknown_fields_and_oversizing_fail(self): lambda v: v['build_info']['schemas'][-1].update(version=True), lambda v: v['build_info'].update(compiled_features=[None]), lambda v: v.update(service_generation=''), - lambda v: v['instance'].update(connection_token='do not expose'), - lambda v: v['instance'].update(capabilities=['x'] * 65), + lambda v: v['instance'].update(capabilities='model.query@1'), + lambda v: v.update(instance=[]), lambda v: v['instance'].update(library_root='x' * 4097), lambda v: v.update(endpoint='http://example.com:8765'), lambda v: v.update(endpoint='http://[::1%lo]:8765'), From e505b88975a913983460e203fb05f84094c8d6b7 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 2 Nov 2026 22:48:06 -0700 Subject: [PATCH 3/3] test(pumas): require the used capability instead of a fixed count The vision gateway fixture is projected from the Pumas DTOs, which now also advertise decision and multimodal_embedding. Pumas responses grow additively, so the browser flow asserts image_to_text is offered rather than an exact capability count. Co-Authored-By: Claude Opus 5.5 --- tests/browser_pumas_typed.cjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/browser_pumas_typed.cjs b/tests/browser_pumas_typed.cjs index ba87eab7..6dfa832f 100644 --- a/tests/browser_pumas_typed.cjs +++ b/tests/browser_pumas_typed.cjs @@ -117,7 +117,8 @@ function launch(command,args,options={}){const child=spawn(command,args,options) assert.equal(await evaluate('document.getElementById("caption-proposal-seed").disabled'),true); await click('caption-proposal-capabilities');await until(()=>evaluate('document.getElementById("caption-proposal-capability-metadata").textContent.includes("image_to_text")')); const visionManifest=JSON.parse(await evaluate('document.getElementById("caption-proposal-capability-metadata").textContent')); - assert.equal(visionManifest.capabilities.capabilities.length,7); + // Pumas responses grow additively; require the capability this flow uses, not a fixed count. + assert.ok(visionManifest.capabilities.capabilities.some(d=>d.capability==='image_to_text')); await evaluate('document.getElementById("caption-proposal-form").requestSubmit()'); const visionJob=await until(async()=> (await request('caption-proposals')).jobs.find(j=>j.source.id===visionSource.id && j.status==='completed')); const visionExact=await request('caption-proposals/'+visionJob.id);