Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/provider-model-discovery-modes.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
"@open-codesign/desktop": patch
"@open-codesign/i18n": patch
"@open-codesign/shared": patch
---

Formalize provider `modelDiscoveryMode` (`models`, `static-hint`, `manual`, `infer-only`) so listing, Settings pickers, imports, and diagnostics no longer treat GET `/models` as universal.
131 changes: 129 additions & 2 deletions apps/desktop/src/main/connection-ipc.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,13 @@ vi.mock('./tls-override', () => ({
}));

import { createHash } from 'node:crypto';
import { resolveListForProviderPlan } from '@open-codesign/shared';
import {
_clearModelsCache,
buildAuthHeaders,
buildAuthHeadersForWire,
CONNECTION_FETCH_TIMEOUT_MS,
type ConnectionTestResponse,
classifyHttpError,
classifyNetworkTarget,
extractIds,
Expand All @@ -27,6 +29,7 @@ import {
getCacheKey,
handleConfigV1TestEndpoint,
handleOllamaV1Probe,
type ModelsListResponse,
normalizeBaseUrl,
normalizeOllamaBaseUrl,
runProviderTest,
Expand All @@ -39,8 +42,6 @@ import { withTlsBypass } from './tls-override';
// network responses without hitting the network.
// ---------------------------------------------------------------------------

import type { ConnectionTestResponse, ModelsListResponse } from './connection-ipc';

// ---------------------------------------------------------------------------
// connection:v1:test test helper
// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -555,6 +556,31 @@ describe('models:v1:list-for-provider input validation', () => {
const result = validateListForProviderInput('claude-code-anthropic');
expect(result).toBeNull();
});

it('respects declared discovery modes instead of always fetching /models', () => {
expect(
resolveListForProviderPlan('chatgpt-codex', {
wire: 'openai-codex-responses',
defaultModel: 'gpt-5.5',
modelsHint: ['gpt-5.5', 'gpt-5.4'],
requiresApiKey: false,
}).action,
).toBe('return');
expect(
resolveListForProviderPlan('glm', {
wire: 'openai-chat',
defaultModel: 'glm-4.6',
capabilities: { modelDiscoveryMode: 'infer-only' },
}),
).toMatchObject({ action: 'return', source: 'local', models: ['glm-4.6'] });
expect(
resolveListForProviderPlan('openai', {
wire: 'openai-chat',
defaultModel: 'gpt-4o',
capabilities: { modelDiscoveryMode: 'models' },
}).action,
).toBe('fetch-remote');
});
});

// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -1188,6 +1214,88 @@ describe('runProviderTest degrade-probe (issue #179)', () => {
});
});

describe('runProviderTest discovery modes (issue #210)', () => {
beforeEach(() => {
vi.useRealTimers();
});

it('infer-only skips GET /models and treats inference success as compatible', async () => {
const { calls, restore } = installFakeFetch((url) => {
if (url.endsWith('/models')) return { status: 404 };
if (url.endsWith('/chat/completions')) return { status: 200, body: { id: 'ok' } };
return { status: 500 };
});
try {
const res = await runProviderTest({
provider: 'glm',
wire: 'openai-chat',
apiKey: 'sk-test',
baseUrl: 'https://open.bigmodel.cn/api/paas/v4',
modelDiscoveryMode: 'infer-only',
});
expect(res.ok).toBe(true);
if (res.ok) {
expect(res.probeMethod).toBe('inference');
expect(res.compatibility).toBe('compatible');
}
expect(calls).toHaveLength(1);
expect(calls[0]?.url).toMatch(/\/chat\/completions$/);
expect(calls.some((c) => c.url.endsWith('/models'))).toBe(false);
} finally {
restore();
}
});

it('manual skips GET /models', async () => {
const { calls, restore } = installFakeFetch((url) => {
if (url.endsWith('/responses')) return { status: 200, body: { id: 'ok' } };
return { status: 404 };
});
try {
const res = await runProviderTest({
provider: 'custom',
wire: 'openai-responses',
apiKey: 'sk-test',
baseUrl: 'https://proxy.example.com/v1',
modelDiscoveryMode: 'manual',
});
expect(res.ok).toBe(true);
if (res.ok) {
expect(res.probeMethod).toBe('inference');
expect(res.compatibility).toBe('compatible');
}
expect(calls.some((c) => c.url.endsWith('/models'))).toBe(false);
} finally {
restore();
}
});

it('models mode still degrade-probes /models 404 as before', async () => {
const { calls, restore } = installFakeFetch((url) => {
if (url.endsWith('/models')) return { status: 404 };
if (url.endsWith('/chat/completions')) return { status: 200, body: { id: 'ok' } };
return { status: 500 };
});
try {
const res = await runProviderTest({
provider: 'glm',
wire: 'openai-chat',
apiKey: 'sk-test',
baseUrl: 'https://open.bigmodel.cn/api/paas/v4',
modelDiscoveryMode: 'models',
});
expect(res.ok).toBe(true);
if (res.ok) {
expect(res.probeMethod).toBe('chat_completion_degraded');
expect(res.compatibility).toBe('degraded');
}
expect(calls[0]?.url).toMatch(/\/models$/);
} finally {
restore();
}
});
});

describe('config:v1:test-endpoint response parsing', () => {
beforeEach(() => {
vi.useRealTimers();
Expand All @@ -1212,6 +1320,25 @@ describe('config:v1:test-endpoint response parsing', () => {
}
});

it('treats GET /models 404 as listing-unavailable rather than a hard failure', async () => {
const { restore } = installFakeFetch(() => ({ status: 404 }));
try {
await expect(
handleConfigV1TestEndpoint({
wire: 'openai-chat',
baseUrl: 'https://open.bigmodel.cn/api/paas/v4',
apiKey: 'sk-test',
}),
).resolves.toEqual({
ok: false,
error: 'not-a-model-endpoint',
message: 'HTTP 404',
});
} finally {
restore();
}
});

it('classifies private and metadata network targets', () => {
expect(classifyNetworkTarget('https://provider.example/v1')).toBe('public');
expect(classifyNetworkTarget('http://localhost:8317')).toBe('loopback');
Expand Down
99 changes: 89 additions & 10 deletions apps/desktop/src/main/connection-ipc.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,15 @@ import {
BUILTIN_PROVIDERS,
CodesignError,
canonicalBaseUrl,
connectionTestProbesModelsEndpoint,
type DiagnosticCategory,
ERROR_CODES,
ensureVersionedBase,
isSupportedOnboardingProvider,
type ProviderEntry,
type ProviderModelDiscoveryMode,
resolveListForProviderPlan,
resolveProviderCapabilities,
type SupportedOnboardingProvider,
stripInferenceEndpointSuffix,
type WireApi,
Expand Down Expand Up @@ -72,6 +76,8 @@ export interface ConnectionTestResult {
ok: true;
/**
* `models` when the standard GET /models probe succeeded.
* `inference` when the provider's discovery mode does not expect /models
* and the real invoke path answered.
* `chat_completion_degraded` when /models 404'd but POST /chat/completions
* proved the openai-chat wire is alive (e.g. Zhipu GLM — no public /models).
* `responses_degraded` when /models 404'd but POST /responses proved the
Expand All @@ -81,6 +87,7 @@ export interface ConnectionTestResult {
*/
probeMethod?:
| 'models'
| 'inference'
| 'chat_completion_degraded'
| 'responses_degraded'
| 'anthropic_messages_degraded';
Expand All @@ -100,12 +107,18 @@ export interface ConnectionTestError {
export type ConnectionTestResponse = ConnectionTestResult | ConnectionTestError;

export type ModelsListResponse =
| { ok: true; models: string[] }
| {
ok: true;
models: string[];
discoveryMode?: ProviderModelDiscoveryMode;
source?: 'remote' | 'static-hint' | 'local';
}
| {
ok: false;
code: 'IPC_BAD_INPUT' | 'NETWORK' | 'HTTP' | 'PARSE';
message: string;
hint: string;
discoveryMode?: ProviderModelDiscoveryMode;
};

function parseConnectionTestPayload(raw: unknown): ConnectionTestPayloadV1 {
Expand Down Expand Up @@ -492,6 +505,9 @@ export interface ActiveProviderCredentials {
builtin?: boolean;
/** Opt-in TLS verification bypass; only honored when `builtin === false`. */
tlsRejectUnauthorized?: boolean;
/** Listing strategy for this provider. Defaults to `models` when omitted so
* existing tests and older callers keep probing GET /models. */
modelDiscoveryMode?: ProviderModelDiscoveryMode;
}

function resolveCredentialsForProvider(
Expand Down Expand Up @@ -539,6 +555,7 @@ function resolveCredentialsForProvider(
apiKey,
baseUrl: entry.baseUrl,
builtin: entry.builtin === true,
modelDiscoveryMode: resolveProviderCapabilities(providerId, entry).modelDiscoveryMode,
...(entry.httpHeaders !== undefined ? { httpHeaders: entry.httpHeaders } : {}),
...(entry.tlsRejectUnauthorized !== undefined
? { tlsRejectUnauthorized: entry.tlsRejectUnauthorized }
Expand Down Expand Up @@ -607,10 +624,15 @@ export async function runProviderTest(
return testChatGPTCodexOAuth();
}

const discoveryMode = creds.modelDiscoveryMode ?? 'models';
const bypass = creds.builtin !== true && creds.tlsRejectUnauthorized === true;
if (!connectionTestProbesModelsEndpoint(discoveryMode)) {
return withTlsBypass(bypass, () => probeDeclaredInference(creds));
}

// Bypass is the per-provider opt-in, force-gated so a tampered config can
// never weaken TLS for built-in providers. Wrapping the whole body covers
// both the GET /models probe and the inner POST inside tryDegradeProbe.
const bypass = creds.builtin !== true && creds.tlsRejectUnauthorized === true;
return withTlsBypass(bypass, async () => {
const { url, normalizedBaseUrl } = buildEndpointForWire(creds.wire, creds.baseUrl);
const headers = buildAuthHeadersForWire(
Expand Down Expand Up @@ -666,6 +688,55 @@ export async function runProviderTest(
});
}

async function probeDeclaredInference(
creds: ActiveProviderCredentials,
): Promise<ConnectionTestResponse> {
if (
creds.wire !== 'openai-chat' &&
creds.wire !== 'openai-responses' &&
creds.wire !== 'anthropic'
) {
return {
ok: false,
code: 'IPC_BAD_INPUT',
message: `Discovery mode does not probe /models for wire "${creds.wire}"`,
hint: 'Use a listed model id or switch this provider to a listing-capable endpoint',
compatibility: 'incompatible',
reasonCategory: 'unknown',
};
}
const { normalizedBaseUrl } = buildEndpointForWire(creds.wire, creds.baseUrl);
const headers = buildAuthHeadersForWire(
creds.wire,
creds.apiKey,
creds.httpHeaders,
creds.baseUrl,
);
const probe = await probeInferenceEndpoint(creds.wire, normalizedBaseUrl, headers);
if (probe.kind === 'pass') {
return { ok: true, probeMethod: 'inference', compatibility: 'compatible' };
}
if (probe.kind === 'network') {
return {
ok: false,
code: 'NETWORK',
message: probe.message,
hint: 'Cannot reach provider inference endpoint',
compatibility: 'incompatible',
reasonCategory: 'unknown',
};
}
const { code, hint } = classifyHttpError(probe.status);
return {
ok: false,
code,
message: `HTTP ${probe.status}`,
hint,
compatibility: 'incompatible',
reasonCategory: connectionCategoryForStatus(probe.status, normalizedBaseUrl),
};
}

async function tryDegradeProbe(
wire: 'openai-chat' | 'openai-responses' | 'anthropic',
normalizedBaseUrl: string,
Expand Down Expand Up @@ -977,19 +1048,24 @@ async function handleModelsV1ListForProvider(raw: unknown): Promise<ModelsListRe
if ('ok' in resolved) return resolved;
const { providerId, entry } = resolved;

// Providers that expose a static hint (e.g. chatgpt-codex, whose /models
// endpoint requires OAuth bearer + ChatGPT-Account-Id headers that this
// keyless discovery path cannot supply) short-circuit with modelsHint.
if (entry.modelsHint !== undefined && entry.modelsHint.length > 0) {
return { ok: true, models: entry.modelsHint };
const plan = resolveListForProviderPlan(providerId, entry);
if (plan.action === 'return') {
return {
ok: true,
models: plan.models,
discoveryMode: plan.discoveryMode,
source: plan.source,
};
}

const keyResult = resolveApiKeyForListing(providerId, entry);
if ('ok' in keyResult) return keyResult;
const { apiKey } = keyResult;

const cached = getCachedModels(providerId, entry.baseUrl, apiKey);
if (cached !== null) return { ok: true, models: cached };
if (cached !== null) {
return { ok: true, models: cached, discoveryMode: 'models', source: 'remote' };
}

const { url } = buildEndpointForWire(entry.wire, entry.baseUrl);
const headers = buildAuthHeadersForWire(entry.wire, apiKey, entry.httpHeaders, entry.baseUrl);
Expand All @@ -1001,8 +1077,11 @@ async function handleModelsV1ListForProvider(raw: unknown): Promise<ModelsListRe
hint: 'Check provider /models endpoint compatibility',
}),
);
if (result.ok) setCachedModels(providerId, entry.baseUrl, apiKey, result.models);
return result;
if (result.ok) {
setCachedModels(providerId, entry.baseUrl, apiKey, result.models);
return { ...result, discoveryMode: 'models', source: 'remote' };
}
return { ...result, discoveryMode: 'models' };
}

async function fetchModelListResponse(
Expand Down
Loading
Loading