refactor(session): move llm stream into layer (#22358)
This commit is contained in:
Kit Langton
@@ -26,6 +26,12 @@ async function getModel(providerID: ProviderID, modelID: ModelID) {
|
||||
)
|
||||
}
|
||||
|
||||
const llm = makeRuntime(LLM.Service, LLM.defaultLayer)
|
||||
|
||||
async function drain(input: LLM.StreamInput) {
|
||||
return llm.runPromise((svc) => svc.stream(input).pipe(Stream.runDrain))
|
||||
}
|
||||
|
||||
describe("session.llm.hasToolCalls", () => {
|
||||
test("returns false for empty messages array", () => {
|
||||
expect(LLM.hasToolCalls([])).toBe(false)
|
||||
@@ -355,20 +361,16 @@ describe("session.llm.stream", () => {
|
||||
model: { providerID: ProviderID.make(providerID), modelID: resolved.id, variant: "high" },
|
||||
} satisfies MessageV2.User
|
||||
|
||||
const stream = await LLM.stream({
|
||||
await drain({
|
||||
user,
|
||||
sessionID,
|
||||
model: resolved,
|
||||
agent,
|
||||
system: ["You are a helpful assistant."],
|
||||
abort: new AbortController().signal,
|
||||
messages: [{ role: "user", content: "Hello" }],
|
||||
tools: {},
|
||||
})
|
||||
|
||||
for await (const _ of stream.fullStream) {
|
||||
}
|
||||
|
||||
const capture = await request
|
||||
const body = capture.body
|
||||
const headers = capture.headers
|
||||
@@ -393,80 +395,6 @@ describe("session.llm.stream", () => {
|
||||
})
|
||||
})
|
||||
|
||||
test("raw stream abort signal cancels provider response body promptly", async () => {
|
||||
const server = state.server
|
||||
if (!server) throw new Error("Server not initialized")
|
||||
|
||||
const providerID = "alibaba"
|
||||
const modelID = "qwen-plus"
|
||||
const fixture = await loadFixture(providerID, modelID)
|
||||
const model = fixture.model
|
||||
const pending = waitStreamingRequest("/chat/completions")
|
||||
|
||||
await using tmp = await tmpdir({
|
||||
init: async (dir) => {
|
||||
await Bun.write(
|
||||
path.join(dir, "opencode.json"),
|
||||
JSON.stringify({
|
||||
$schema: "https://opencode.ai/config.json",
|
||||
enabled_providers: [providerID],
|
||||
provider: {
|
||||
[providerID]: {
|
||||
options: {
|
||||
apiKey: "test-key",
|
||||
baseURL: `${server.url.origin}/v1`,
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
)
|
||||
},
|
||||
})
|
||||
|
||||
await Instance.provide({
|
||||
directory: tmp.path,
|
||||
fn: async () => {
|
||||
const resolved = await getModel(ProviderID.make(providerID), ModelID.make(model.id))
|
||||
const sessionID = SessionID.make("session-test-raw-abort")
|
||||
const agent = {
|
||||
name: "test",
|
||||
mode: "primary",
|
||||
options: {},
|
||||
permission: [{ permission: "*", pattern: "*", action: "allow" }],
|
||||
} satisfies Agent.Info
|
||||
const user = {
|
||||
id: MessageID.make("user-raw-abort"),
|
||||
sessionID,
|
||||
role: "user",
|
||||
time: { created: Date.now() },
|
||||
agent: agent.name,
|
||||
model: { providerID: ProviderID.make(providerID), modelID: resolved.id },
|
||||
} satisfies MessageV2.User
|
||||
|
||||
const ctrl = new AbortController()
|
||||
const result = await LLM.stream({
|
||||
user,
|
||||
sessionID,
|
||||
model: resolved,
|
||||
agent,
|
||||
system: ["You are a helpful assistant."],
|
||||
abort: ctrl.signal,
|
||||
messages: [{ role: "user", content: "Hello" }],
|
||||
tools: {},
|
||||
})
|
||||
|
||||
const iter = result.fullStream[Symbol.asyncIterator]()
|
||||
await pending.request
|
||||
await iter.next()
|
||||
ctrl.abort()
|
||||
|
||||
await Promise.race([pending.responseCanceled, timeout(500)])
|
||||
await Promise.race([pending.requestAborted, timeout(500)]).catch(() => undefined)
|
||||
await iter.return?.()
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
test("service stream cancellation cancels provider response body promptly", async () => {
|
||||
const server = state.server
|
||||
if (!server) throw new Error("Server not initialized")
|
||||
@@ -518,8 +446,7 @@ describe("session.llm.stream", () => {
|
||||
} satisfies MessageV2.User
|
||||
|
||||
const ctrl = new AbortController()
|
||||
const { runPromiseExit } = makeRuntime(LLM.Service, LLM.defaultLayer)
|
||||
const run = runPromiseExit(
|
||||
const run = llm.runPromiseExit(
|
||||
(svc) =>
|
||||
svc
|
||||
.stream({
|
||||
@@ -610,14 +537,13 @@ describe("session.llm.stream", () => {
|
||||
tools: { question: true },
|
||||
} satisfies MessageV2.User
|
||||
|
||||
const stream = await LLM.stream({
|
||||
await drain({
|
||||
user,
|
||||
sessionID,
|
||||
model: resolved,
|
||||
agent,
|
||||
permission: [{ permission: "question", pattern: "*", action: "allow" }],
|
||||
system: ["You are a helpful assistant."],
|
||||
abort: new AbortController().signal,
|
||||
messages: [{ role: "user", content: "Hello" }],
|
||||
tools: {
|
||||
question: tool({
|
||||
@@ -628,9 +554,6 @@ describe("session.llm.stream", () => {
|
||||
},
|
||||
})
|
||||
|
||||
for await (const _ of stream.fullStream) {
|
||||
}
|
||||
|
||||
const capture = await request
|
||||
const tools = capture.body.tools as Array<{ function?: { name?: string } }> | undefined
|
||||
expect(tools?.some((item) => item.function?.name === "question")).toBe(true)
|
||||
@@ -728,20 +651,16 @@ describe("session.llm.stream", () => {
|
||||
model: { providerID: ProviderID.make("openai"), modelID: resolved.id, variant: "high" },
|
||||
} satisfies MessageV2.User
|
||||
|
||||
const stream = await LLM.stream({
|
||||
await drain({
|
||||
user,
|
||||
sessionID,
|
||||
model: resolved,
|
||||
agent,
|
||||
system: ["You are a helpful assistant."],
|
||||
abort: new AbortController().signal,
|
||||
messages: [{ role: "user", content: "Hello" }],
|
||||
tools: {},
|
||||
})
|
||||
|
||||
for await (const _ of stream.fullStream) {
|
||||
}
|
||||
|
||||
const capture = await request
|
||||
const body = capture.body
|
||||
|
||||
@@ -847,13 +766,12 @@ describe("session.llm.stream", () => {
|
||||
model: { providerID: ProviderID.make("openai"), modelID: resolved.id },
|
||||
} satisfies MessageV2.User
|
||||
|
||||
const stream = await LLM.stream({
|
||||
await drain({
|
||||
user,
|
||||
sessionID,
|
||||
model: resolved,
|
||||
agent,
|
||||
system: ["You are a helpful assistant."],
|
||||
abort: new AbortController().signal,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
@@ -871,9 +789,6 @@ describe("session.llm.stream", () => {
|
||||
tools: {},
|
||||
})
|
||||
|
||||
for await (const _ of stream.fullStream) {
|
||||
}
|
||||
|
||||
const capture = await request
|
||||
expect(capture.url.pathname.endsWith("/responses")).toBe(true)
|
||||
},
|
||||
@@ -972,20 +887,16 @@ describe("session.llm.stream", () => {
|
||||
model: { providerID: ProviderID.make("minimax"), modelID: ModelID.make("MiniMax-M2.5") },
|
||||
} satisfies MessageV2.User
|
||||
|
||||
const stream = await LLM.stream({
|
||||
await drain({
|
||||
user,
|
||||
sessionID,
|
||||
model: resolved,
|
||||
agent,
|
||||
system: ["You are a helpful assistant."],
|
||||
abort: new AbortController().signal,
|
||||
messages: [{ role: "user", content: "Hello" }],
|
||||
tools: {},
|
||||
})
|
||||
|
||||
for await (const _ of stream.fullStream) {
|
||||
}
|
||||
|
||||
const capture = await request
|
||||
const body = capture.body
|
||||
|
||||
@@ -1073,20 +984,16 @@ describe("session.llm.stream", () => {
|
||||
model: { providerID: ProviderID.make(providerID), modelID: resolved.id },
|
||||
} satisfies MessageV2.User
|
||||
|
||||
const stream = await LLM.stream({
|
||||
await drain({
|
||||
user,
|
||||
sessionID,
|
||||
model: resolved,
|
||||
agent,
|
||||
system: ["You are a helpful assistant."],
|
||||
abort: new AbortController().signal,
|
||||
messages: [{ role: "user", content: "Hello" }],
|
||||
tools: {},
|
||||
})
|
||||
|
||||
for await (const _ of stream.fullStream) {
|
||||
}
|
||||
|
||||
const capture = await request
|
||||
const body = capture.body
|
||||
const config = body.generationConfig as
|
||||
|
||||
Reference in New Issue
Block a user