Repository navigation
Expand file tree
/
Copy pathqueue.test.mjs
More file actions
269 lines (251 loc) · 13.6 KB
/
Copy pathqueue.test.mjs
File metadata and controls
269 lines (251 loc) · 13.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
// Qoder's queue (code 10605) is waited out as Qoder's own client waits it
// (qoder-agent-sdk's worker in Qoder.app 0.4.3, "[ModelQueue]"): it asks
// /api/v2/service/ask/queue/status with the request's requestSetId, the
// model and the queueType until isQueued is false, waits that state's
// retryAfterSeconds and sends the request again (its request_set_id, a new
// request_id). wallance on Discord:
// Qoder's free dfmodel answered "Qoder queued the request for dfmodel (this
// plan's queue: p3); try again in 30s" through magpie, while Qoder's client
// answered, slowly. The refusal is the shape of the real one in
// testdata/queued-10605.json with isQueued true, as wallance's message
// says; the status replies are that shape too, not captured bytes.
//
// The clock and the pauses are the test's own (QUEUE.now, QUEUE.pause): a
// pause moves the clock on at once, and says how long it was asked to be.
import { afterEach, beforeEach, expect, test } from "bun:test"
import { QoderAuthPlugin, _internal } from "./index.mjs"
const { QUEUE } = _internal
const real = globalThis.fetch
const saved = { ...QUEUE }
let clock, pauses
beforeEach(() => {
clock = 1_000_000
pauses = []
QUEUE.now = () => clock
QUEUE.pause = async (ms, signal) => {
if (signal?.aborted) throw signal.reason
pauses.push(ms)
clock += ms
}
})
afterEach(() => {
globalThis.fetch = real
Object.assign(QUEUE, saved)
})
const API_CHAT = "https://api3.qoder.sh/v1/chat/completions"
const ask = (signal) => ({ method: "POST", body: JSON.stringify({ model: "dfmodel", messages: [{ role: "user", content: "hi" }] }), signal })
const LISTING = { chat: [{ key: "dfmodel", display_name: "DeepSeek Flash", enable: true }] }
const json = (v, status = 200) => new Response(JSON.stringify(v), { status, headers: { "Content-Type": "application/json" } })
const QUEUED = { isQueued: true, modelKey: "dfmodel", queueCount: 3, queueType: "p3", retryAfterSeconds: 30, serviceAvailable: true, waitTime: 20 }
const refused = (q = QUEUED) => json({ code: "10605", message: JSON.stringify(q) }, 403)
const line = (body, statusCodeValue = 200) => `data:${JSON.stringify({ headers: {}, body: typeof body === "string" ? body : JSON.stringify(body), statusCodeValue })}\n\n`
const answered = () =>
new Response(line({ choices: [{ delta: { content: "hello" } }] }) + line({ choices: [{ delta: {}, finish_reason: "stop" }] }) + line("[DONE]"), {
headers: { "Content-Type": "text/event-stream" },
})
const status = (st) => json({ success: true, data: st })
// a loader whose chats are answered by chats(), in turn, and status asks by
// statuses(); what was sent is kept
async function loader({ chats, statuses }) {
const auth = { type: "oauth", access: "jt", refresh: "rt", expires: Date.now() + 3_600_000, accountId: "one@x", uid: "u1", machineId: "m1" }
const sent = { chats: [], statuses: [] }
globalThis.fetch = async (url, init = {}) => {
const u = new URL(String(url))
if (u.pathname.endsWith("/model/list")) return json(LISTING)
if (u.pathname.endsWith("/queue/status")) {
sent.statuses.push({ url: u, headers: init.headers })
return statuses(sent.statuses.length)
}
sent.chats.push(JSON.parse(_internal.decodeBody(init.body)))
return chats(sent.chats.length)
}
const hooks = await QoderAuthPlugin({ client: { auth: { set: async () => {} } } })
const l = await hooks.auth.loader(async () => auth)
return { fetch: l.fetch, sent }
}
test("a queued request waits its turn in Qoder's queue and is sent again, in the same request set under a new request id, once the queue lets it through", async () => {
const { fetch, sent } = await loader({
chats: (n) => (n === 1 ? refused() : answered()),
statuses: (n) => status(n < 3 ? { ...QUEUED, queueCount: 3 - n, retryAfterSeconds: 12 } : { isQueued: false, modelKey: "dfmodel", queueType: "p3", retryAfterSeconds: 1, serviceAvailable: true }),
})
const res = await fetch(API_CHAT, ask())
expect(res.status).toBe(200)
expect((await res.json()).choices[0].message.content).toBe("hello")
// asked at once, then after each state's retryAfterSeconds; once through,
// its own retryAfterSeconds before the request goes again
expect(pauses).toEqual([12_000, 12_000, 1_000])
expect(sent.chats.length).toBe(2)
// the queue knows it by its request_set_id; the resend is a new call in
// it, a new request_id and chat_record_id, as Qoder's client sends it
// (o4i before each resend)
expect(sent.chats[1].request_set_id).toBe(sent.chats[0].request_set_id)
expect(sent.chats[1].request_id).not.toBe(sent.chats[0].request_id)
expect(sent.chats[1].chat_record_id).toBe(sent.chats[1].request_id)
expect(sent.chats[1].session_id).toBe(sent.chats[0].session_id)
expect(sent.chats[1].messages).toEqual(sent.chats[0].messages)
const first = sent.statuses[0]
expect(first.url.origin + first.url.pathname).toBe("https://api3.qoder.sh/algo/api/v2/service/ask/queue/status")
expect(Object.fromEntries(first.url.searchParams)).toEqual({ requestSetId: sent.chats[0].request_set_id, modelKey: "dfmodel", queueType: "p3" })
expect(first.headers["X-Request-ID"]).toBe(sent.chats[0].request_id)
expect(first.headers["X-Session-ID"]).toBe(sent.chats[0].session_id)
expect(first.headers.Authorization).toStartWith("Bearer COSY.")
})
test("a refusal's stream is closed before the wait, and the status asks after a resend name the resend", async () => {
// Qoder keeps a 10605 stream open after its one event (seen on a real
// qoder.com account, plugins#427): it is closed, not held through the
// wait and the resend
const signals = []
const auth = { type: "oauth", access: "jt", refresh: "rt", expires: Date.now() + 3_600_000, accountId: "one@x", uid: "u1", machineId: "m1" }
const chats = []
const statuses = []
globalThis.fetch = async (url, init = {}) => {
const u = new URL(String(url))
if (u.pathname.endsWith("/model/list")) return json(LISTING)
if (u.pathname.endsWith("/queue/status")) {
statuses.push(init.headers["X-Request-ID"])
return status(statuses.length % 2 ? QUEUED : { isQueued: false, serviceAvailable: true })
}
chats.push(JSON.parse(_internal.decodeBody(init.body)))
signals.push(init.signal)
if (chats.length === 3) return answered()
const ev = new TextEncoder().encode(line({ code: "10605", message: JSON.stringify(QUEUED) }, 403))
return new Response(new ReadableStream({ start: (c) => c.enqueue(ev) }), { headers: { "Content-Type": "text/event-stream" } })
}
const hooks = await QoderAuthPlugin({ client: { auth: { set: async () => {} } } })
const l = await hooks.auth.loader(async () => auth)
const res = await l.fetch(API_CHAT, ask())
expect(res.status).toBe(200)
expect((await res.json()).choices[0].message.content).toBe("hello")
expect(chats.length).toBe(3)
expect(signals.map((s) => s.aborted)).toEqual([true, true, false])
expect(new Set(chats.map((c) => c.request_id)).size).toBe(3)
expect(new Set(chats.map((c) => c.request_set_id)).size).toBe(1)
expect(statuses).toEqual([chats[0].request_id, chats[0].request_id, chats[1].request_id, chats[1].request_id])
})
test("the queue's refusal in the stream is waited out the same way", async () => {
const { fetch, sent } = await loader({
chats: (n) =>
n === 1
? new Response(line({ code: "10605", message: JSON.stringify(QUEUED) }, 403), { headers: { "Content-Type": "text/event-stream" } })
: answered(),
statuses: () => status({ isQueued: false, serviceAvailable: true }),
})
const res = await fetch(API_CHAT, ask())
expect(res.status).toBe(200)
expect((await res.json()).choices[0].message.content).toBe("hello")
expect(sent.chats.length).toBe(2)
})
test("a wait held past the budget is the 503 with Retry-After it was, the request not sent again", async () => {
const { fetch, sent } = await loader({ chats: () => refused(), statuses: () => status(QUEUED) })
const res = await fetch(API_CHAT, ask())
expect(res.status).toBe(503)
expect(res.headers.get("Retry-After")).toBe("30")
expect((await res.json()).error.message).toBe("Qoder queued the request for dfmodel (this plan's queue: p3, 3 ahead); try again in 30s")
expect(sent.chats.length).toBe(1)
// 30s pauses, the last cut to what was left of the budget
expect(pauses.reduce((a, b) => a + b, 0)).toBe(QUEUE.budget)
expect(Math.max(...pauses)).toBe(30_000)
})
// With another candidate after it (magpie's X-Magpie-Fallback: the model's
// next account or provider, a routing group's next member), a queued
// request isn't waited out: its 503 goes back at once, with where it stood
// in the queue, and magpie asks the next (axuhere on X: a free model held
// at "预计等待超过 10 分钟" while another provider could answer). The last
// candidate isn't told, and waits as Qoder's client does (above).
test("told another candidate follows, a queued request is answered at once with the queue's reason", async () => {
const { fetch, sent } = await loader({ chats: () => refused(), statuses: () => status(QUEUED) })
const res = await fetch(API_CHAT, { ...ask(), headers: { "x-magpie-fallback": "1" } })
expect(res.status).toBe(503)
expect(res.headers.get("Retry-After")).toBe("30")
expect((await res.json()).error.message).toBe("Qoder queued the request for dfmodel (this plan's queue: p3, 3 ahead); try again in 30s")
expect(sent.chats.length).toBe(1)
expect(sent.statuses.length).toBe(0)
expect(pauses).toEqual([])
})
test("the pause is the queue's retryAfterSeconds held to 0.5s..30s, and 30s when it says none", async () => {
const states = [
{ ...QUEUED, retryAfterSeconds: 0.1 },
{ ...QUEUED, retryAfterSeconds: 300 },
{ ...QUEUED, retryAfterSeconds: undefined },
{ isQueued: false },
]
const { fetch } = await loader({ chats: (n) => (n === 1 ? refused() : answered()), statuses: (n) => status(states[n - 1]) })
expect((await fetch(API_CHAT, ask())).status).toBe(200)
expect(pauses).toEqual([500, 30_000, 30_000])
})
test("service unavailable keeps waiting; the queue saying nothing of itself, or no queue there (404), ends the wait at once", async () => {
let r = await loader({
chats: (n) => (n === 1 ? refused() : answered()),
statuses: (n) => status(n === 1 ? { isQueued: false, serviceAvailable: false, retryAfterSeconds: 5 } : { isQueued: false }),
})
// isQueued false comes first, as in Qoder's client: through, after 5s
expect((await r.fetch(API_CHAT, ask())).status).toBe(200)
expect(pauses).toEqual([5_000])
pauses = []
r = await loader({
chats: (n) => (n === 1 ? refused() : answered()),
statuses: (n) => status(n === 1 ? { serviceAvailable: false, retryAfterSeconds: 2 } : { isQueued: false }),
})
expect((await r.fetch(API_CHAT, ask())).status).toBe(200)
expect(pauses).toEqual([2_000])
for (const reply of [() => status({ nothing: 1 }), () => new Response("not found", { status: 404 })]) {
pauses = []
r = await loader({ chats: () => refused(), statuses: reply })
expect((await r.fetch(API_CHAT, ask())).status).toBe(503)
expect(r.sent.statuses.length).toBe(1)
expect(r.sent.chats.length).toBe(1)
}
})
test("three failed status asks in a row end the wait; one between good ones doesn't", async () => {
let r = await loader({ chats: () => refused(), statuses: () => new Response("boom", { status: 500 }) })
expect((await r.fetch(API_CHAT, ask())).status).toBe(503)
expect(r.sent.statuses.length).toBe(3)
// a failed ask's pause isn't the last state's: 30s
expect(pauses).toEqual([30_000, 30_000])
pauses = []
const replies = [() => new Response("", { status: 502 }), () => status(QUEUED), () => new Response("", { status: 502 }), () => new Response("", { status: 502 }), () => status({ isQueued: false })]
r = await loader({ chats: (n) => (n === 1 ? refused() : answered()), statuses: (n) => replies[n - 1]() })
expect((await r.fetch(API_CHAT, ask())).status).toBe(200)
expect(r.sent.statuses.length).toBe(5)
})
test("the queue is waited out 10 times for one request at most", async () => {
QUEUE.budget = 24 * 60 * 60 * 1000
const { fetch, sent } = await loader({ chats: () => refused(), statuses: () => status({ isQueued: false }) })
const res = await fetch(API_CHAT, ask())
expect(res.status).toBe(503)
expect(sent.chats.length).toBe(11)
})
test("a cancelled request stops waiting: no status asked and no request sent after", async () => {
const stop = new AbortController()
const { fetch, sent } = await loader({
chats: () => refused(),
statuses: (n) => {
if (n === 2) stop.abort(new DOMException("the client went away", "AbortError"))
return status(QUEUED)
},
})
const res = await fetch(API_CHAT, ask(stop.signal))
expect(res.status).toBe(502)
expect(sent.statuses.length).toBe(2)
expect(sent.chats.length).toBe(1)
})
test("the real pause ends early when the request is cancelled", async () => {
QUEUE.pause = saved.pause
const stop = new AbortController()
const p = QUEUE.pause(60_000, stop.signal)
stop.abort(new DOMException("gone", "AbortError"))
await expect(p).rejects.toThrow("gone")
})
test("the state is found where Qoder's client finds it", () => {
expect(_internal.queueState(JSON.stringify({ data: { isQueued: true, queueType: "p3", retryAfterSeconds: "7" } }))).toEqual({
isQueued: true,
serviceAvailable: undefined,
modelKey: undefined,
queueType: "p3",
retryAfterSeconds: 7,
})
expect(_internal.queueState(JSON.stringify({ code: "200", message: JSON.stringify({ isQueued: false }) }))?.isQueued).toBe(false)
expect(_internal.queueState(JSON.stringify({ result: JSON.stringify({ serviceAvailable: false }) }))?.serviceAvailable).toBe(false)
expect(_internal.queueState("not json")).toBeUndefined()
expect(_internal.queueState(JSON.stringify({ data: { queueCount: 1 } }))).toBeUndefined()
})