From eab9a4f5642bf62e2c8ab077139f0650e73e4900 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 14 May 2026 09:12:43 -0400 Subject: [PATCH] llm provider and test renderer --- ...roperty-based-tui-testing-working-notes.md | 27 ++++ .../specs/property-based-tui-testing.md | 25 +-- packages/opencode/src/cli/cmd/tui/app.tsx | 10 +- .../opencode/src/cli/cmd/tui/simulation.ts | 30 ++++ packages/opencode/src/cli/cmd/tui/thread.ts | 54 ++++--- .../server/routes/instance/httpapi/server.ts | 3 +- .../routes/instance/httpapi/simulation.ts | 7 +- .../src/testing/simulation/provider.ts | 142 ++++++++++++++++++ .../src/testing/simulation/service.ts | 67 ++++++++- .../test/testing/simulation/service.test.ts | 86 ++++++++++- 10 files changed, 409 insertions(+), 42 deletions(-) create mode 100644 packages/opencode/specs/property-based-tui-testing-working-notes.md create mode 100644 packages/opencode/src/cli/cmd/tui/simulation.ts create mode 100644 packages/opencode/src/testing/simulation/provider.ts diff --git a/packages/opencode/specs/property-based-tui-testing-working-notes.md b/packages/opencode/specs/property-based-tui-testing-working-notes.md new file mode 100644 index 000000000..6e882c06f --- /dev/null +++ b/packages/opencode/specs/property-based-tui-testing-working-notes.md @@ -0,0 +1,27 @@ +# Property-Based TUI Testing Working Notes + +## Mock LLM Provider + +- `SessionPrompt` gets model metadata through `Provider.Service.getModel(...)`. +- Actual generation is routed through `LLM.Service` / `streamText(...)`, so the first mock should return an AI SDK `LanguageModelV3` from the provider service. +- The normal `Provider.layer` builds providers from config/models.dev/plugin state. For first pass, the simulated graph can replace `Provider.Service` with a smaller simulation provider service instead of trying to flow through provider config. +- Control state should own an ordered LLM script queue. The provider/model should consume from that queue when the AI SDK calls the language model. +- First version can support text-only output. Tool calls and stream chunk fidelity can come next. +- Missing script should fail loudly with a typed simulation error, not silently return an empty assistant message. +- Implemented `SimulationProvider.layer`, replacing `Provider.Service` in `createSimulatedRoutes`. +- The provider exposes provider `simulation` and model `mock`. +- `doGenerate` and `doStream` both consume one queued script through `Simulation.Service.nextLLM()`. +- Current script support: `text`, `thinking` (treated as text for now), and `error`. +- Snapshot currently records `llmQueued` and `llmConsumed`, not per-step details yet. + +## OpenTUI Fake Renderer + +- OpenTUI Solid exposes `testRender(...)` from `@opentui/solid`. +- The lower-level core API is `createTestRenderer(...)` from `@opentui/core/testing`. +- `createTestRenderer(...)` returns `renderer`, `mockInput`, `mockMouse`, `renderOnce`, `captureCharFrame`, `captureSpans`, and `resize`. +- `captureCharFrame()` is the simple screen-buffer string API used heavily in OpenTUI snapshots. +- `captureSpans()` returns structured lines/spans plus cursor position, which is a better starting point for visible element discovery than parsing raw characters. +- `mockInput` supports interactions like `typeText`, `pressEnter`, and `pressArrow`. +- Implemented `TuiSimulation.createSimulationRenderer(...)` beside `thread.ts`. It creates a test renderer and exposes `renderOnce`, `screen`, `spans`, and `destroy`. +- `thread.ts` checks `OPENCODE_SIMULATION`, creates the fake renderer there, starts the normal worker/backend, and passes the renderer into `tui(...)`. +- `tui(...)` now accepts an injected `CliRenderer`, test mode, and an `onReady` callback. Production still creates the real renderer. diff --git a/packages/opencode/specs/property-based-tui-testing.md b/packages/opencode/specs/property-based-tui-testing.md index c089778a4..6abe7f750 100644 --- a/packages/opencode/specs/property-based-tui-testing.md +++ b/packages/opencode/specs/property-based-tui-testing.md @@ -199,7 +199,9 @@ Implementation shape: - First pass uses a raw route wrapper at `packages/opencode/src/server/routes/instance/httpapi/simulation.ts` to avoid SDK regeneration while the API shape is still moving. - Current control service can reset state, seed filesystem files, register static network responses, and return a snapshot. - Register/configure a local mock provider/model through the normal provider path. -- The mock model reads scripts from simulation control state. +- The simulated route graph replaces `Provider.Service` with `SimulationProvider.layer`. +- The mock model reads queued scripts from simulation control state. +- Current mock provider supports text/thinking/error actions for the first step only. Tool calls and multi-round step selection are still pending. - No JSON-in-prompt fallback. - Missing script means typed simulation error. @@ -236,10 +238,11 @@ Todos: - [x] Add simulation control state and reset semantics. - [x] Add gated simulation endpoints for reset, filesystem seed, network register, and snapshot. - [x] Decide raw route vs typed HttpApi route. Raw route for first pass; no SDK regeneration yet. -- [ ] Implement mock provider/model on the normal provider path. -- [ ] Port the useful stream chunk behavior from the old branch to the current AI SDK interface. -- [ ] Make missing scripts fail with a typed simulation error. -- [ ] Record consumed script step in simulation snapshot. +- [x] Implement mock provider/model on the normal provider path. +- [x] Make missing scripts fail with a typed simulation error. +- [x] Record consumed script count in simulation snapshot. +- [ ] Support tool-call script actions. +- [ ] Support multi-step script selection after tool result rounds. - [ ] Verify `session.prompt_async` exercises real `SessionPrompt` and `SessionProcessor`. ## OpenTUI Fake Renderer And Interactable Elements @@ -255,6 +258,8 @@ Known starting points: Implementation shape: - Add a renderer factory/testing hook to `tui(...)` so tests can pass a fake renderer. +- Current first pass checks `OPENCODE_SIMULATION` in `cli/cmd/tui/thread.ts`, starts the normal worker/backend, and injects an OpenTUI test renderer into `tui(...)`. +- Fake renderer setup lives in `cli/cmd/tui/simulation.ts` and returns `renderOnce`, `screen`, and `spans` helpers for the thread-side simulation runner. - Do not render to a real terminal in simulation mode. - Investigate OpenTUI APIs for walking the render tree and extracting focusable/clickable/editable elements. - Investigate OpenTUI APIs for reading the screen buffer from the fake renderer. @@ -262,11 +267,11 @@ Implementation shape: Todos: -- [ ] Inspect `@opentui/core/testing` `createTestRenderer` capabilities. -- [ ] Inspect `@opentui/solid` `testRender` capabilities. -- [ ] Determine how to get a screen buffer string/snapshot from the fake renderer. -- [ ] Determine how to iterate renderables and identify interactable elements. -- [ ] Add a minimal renderer factory override to `tui(...)` or app startup. +- [x] Inspect `@opentui/core/testing` `createTestRenderer` capabilities. +- [x] Inspect `@opentui/solid` `testRender` capabilities. +- [x] Determine how to get a screen buffer string/snapshot from the fake renderer. +- [x] Determine first structured capture API for interactable discovery: `captureSpans()`. +- [x] Add a minimal renderer factory override to `tui(...)` or app startup. - [ ] Expose prompt ref, route, sync state, keymap, and renderer to the simulation harness. - [ ] Verify TUI starts in fake renderer with no real terminal output. - [ ] Verify screen buffer can be captured after a render. diff --git a/packages/opencode/src/cli/cmd/tui/app.tsx b/packages/opencode/src/cli/cmd/tui/app.tsx index 19a15a26e..e47d10f05 100644 --- a/packages/opencode/src/cli/cmd/tui/app.tsx +++ b/packages/opencode/src/cli/cmd/tui/app.tsx @@ -3,7 +3,7 @@ import { createDefaultOpenTuiKeymap } from "@opentui/keymap/opentui" import * as Clipboard from "@tui/util/clipboard" import * as Selection from "@tui/util/selection" import * as TuiAudio from "@tui/util/audio" -import { createCliRenderer, MouseButton, type CliRendererConfig } from "@opentui/core" +import { createCliRenderer, MouseButton, type CliRenderer, type CliRendererConfig } from "@opentui/core" import { RouteProvider, useRoute } from "@tui/context/route" import { Switch, @@ -165,6 +165,9 @@ export function tui(input: { fetch?: typeof fetch headers?: RequestInit["headers"] events?: EventSource + renderer?: CliRenderer + mode?: "dark" | "light" + onReady?: (ctx: { renderer: CliRenderer }) => void }) { // promise to prevent immediate exit // oxlint-disable-next-line no-async-promise-executor -- intentional: async executor used for sequential setup before resolve @@ -182,10 +185,10 @@ export function tui(input: { TuiAudio.dispose() } - const renderer = await createCliRenderer(rendererConfig(input.config)) + const renderer = input.renderer ?? (await createCliRenderer(rendererConfig(input.config))) // Prewarm palette before ThemeProvider mounts so `system` theme avoids a first-paint fallback flash. void renderer.getPalette({ size: 16 }).catch(() => undefined) - const mode = (await renderer.waitForThemeMode(1000)) ?? "dark" + const mode = input.mode ?? (await renderer.waitForThemeMode(1000)) ?? "dark" const keymap = createDefaultOpenTuiKeymap(renderer) const offKeymap = registerOpencodeKeymap(keymap, renderer, input.config) @@ -256,6 +259,7 @@ export function tui(input: { ) }, renderer) + input.onReady?.({ renderer }) }) } diff --git a/packages/opencode/src/cli/cmd/tui/simulation.ts b/packages/opencode/src/cli/cmd/tui/simulation.ts new file mode 100644 index 000000000..7bd777423 --- /dev/null +++ b/packages/opencode/src/cli/cmd/tui/simulation.ts @@ -0,0 +1,30 @@ +import type { CliRenderer } from "@opentui/core" +import type { CapturedFrame } from "@opentui/core" + +export interface SimulationRenderer { + readonly renderer: CliRenderer + readonly renderOnce: () => Promise + readonly screen: () => string + readonly spans: () => CapturedFrame + readonly destroy: () => void +} + +export async function createSimulationRenderer(): Promise { + const { createTestRenderer } = await import("@opentui/core/testing") + const setup = await createTestRenderer({ + width: Number(process.env.OPENCODE_SIMULATION_TUI_WIDTH) || 100, + height: Number(process.env.OPENCODE_SIMULATION_TUI_HEIGHT) || 40, + screenMode: "main-screen", + consoleMode: "disabled", + }) + + return { + renderer: setup.renderer, + renderOnce: setup.renderOnce, + screen: setup.captureCharFrame, + spans: setup.captureSpans, + destroy: () => setup.renderer.destroy(), + } +} + +export * as TuiSimulation from "./simulation" diff --git a/packages/opencode/src/cli/cmd/tui/thread.ts b/packages/opencode/src/cli/cmd/tui/thread.ts index 7230dae16..981ccad51 100644 --- a/packages/opencode/src/cli/cmd/tui/thread.ts +++ b/packages/opencode/src/cli/cmd/tui/thread.ts @@ -21,6 +21,8 @@ import { sanitizedProcessEnv, } from "@opencode-ai/core/util/opencode-process" import { validateSession } from "./validate-session" +import { Flag } from "@opencode-ai/core/flag/flag" +import { TuiSimulation } from "./simulation" declare global { const OPENCODE_WORKER_PATH: string @@ -229,26 +231,38 @@ export const TuiThreadCommand = cmd({ try { const { tui } = await import("./app") - await tui({ - url: transport.url, - async onSnapshot() { - const tui = writeHeapSnapshot("tui.heapsnapshot") - const server = await client.call("snapshot", undefined) - return [tui, server] - }, - config, - directory: cwd, - fetch: transport.fetch, - events: transport.events, - args: { - continue: args.continue, - sessionID: args.session, - agent: args.agent, - model: args.model, - prompt, - fork: args.fork, - }, - }) + const simulationRenderer = Flag.OPENCODE_SIMULATION ? await TuiSimulation.createSimulationRenderer() : undefined + try { + await tui({ + url: transport.url, + async onSnapshot() { + const tui = writeHeapSnapshot("tui.heapsnapshot") + const server = await client.call("snapshot", undefined) + return [tui, server] + }, + config, + directory: cwd, + fetch: transport.fetch, + events: transport.events, + renderer: simulationRenderer?.renderer, + mode: simulationRenderer ? "dark" : undefined, + onReady: simulationRenderer + ? async () => { + await simulationRenderer.renderOnce() + } + : undefined, + args: { + continue: args.continue, + sessionID: args.session, + agent: args.agent, + model: args.model, + prompt, + fork: args.fork, + }, + }) + } finally { + simulationRenderer?.destroy() + } } finally { await stop() } diff --git a/packages/opencode/src/server/routes/instance/httpapi/server.ts b/packages/opencode/src/server/routes/instance/httpapi/server.ts index d23f720a2..938cae6cb 100644 --- a/packages/opencode/src/server/routes/instance/httpapi/server.ts +++ b/packages/opencode/src/server/routes/instance/httpapi/server.ts @@ -64,6 +64,7 @@ import { Workspace } from "@/control-plane/workspace" import { SimulationFileSystem } from "@/testing/simulation/filesystem" import { SimulationNetwork } from "@/testing/simulation/network" import { SimulationNetworkRoutes } from "@/testing/simulation/network-routes" +import { SimulationProvider } from "@/testing/simulation/provider" import { Simulation } from "@/testing/simulation/service" import { CorsConfig, isAllowedCorsOrigin, type CorsOptions } from "@/server/cors" import { serveUIEffect } from "@/server/shared/ui" @@ -284,7 +285,7 @@ export function createSimulatedRoutes(corsOptions?: CorsOptions): ReturnType ) yield* router.add("POST", "/experimental/simulation/llm/enqueue", () => - json(Effect.succeed({ ok: false, skipped: "mock provider scripts are not implemented yet" })), + json( + Effect.gen(function* () { + const input = yield* HttpServerRequest.schemaBodyJson(Simulation.LLMEnqueueInput) + return yield* simulation.enqueueLLM(input) + }), + ), ) yield* router.add("GET", "/experimental/simulation/snapshot", () => json(simulation.snapshot())) diff --git a/packages/opencode/src/testing/simulation/provider.ts b/packages/opencode/src/testing/simulation/provider.ts new file mode 100644 index 000000000..9f7121ec7 --- /dev/null +++ b/packages/opencode/src/testing/simulation/provider.ts @@ -0,0 +1,142 @@ +import type { LanguageModelV3, LanguageModelV3CallOptions, LanguageModelV3FinishReason } from "@ai-sdk/provider" +import { Effect, Layer } from "effect" +import { Provider } from "@/provider/provider" +import { ModelID, ProviderID } from "@/provider/schema" +import { Simulation, type LLMScript } from "./service" + +const providerID = ProviderID.make("simulation") +const modelID = ModelID.make("mock") + +const model: Provider.Model = { + id: modelID, + providerID, + api: { id: modelID, url: "simulation://mock", npm: "simulation" }, + name: "Simulation Mock", + capabilities: { + temperature: true, + reasoning: true, + attachment: false, + toolcall: false, + input: { text: true, audio: false, image: false, video: false, pdf: false }, + output: { text: true, audio: false, image: false, video: false, pdf: false }, + interleaved: false, + }, + cost: { input: 0, output: 0, cache: { read: 0, write: 0 } }, + limit: { context: 128_000, output: 32_000 }, + status: "active", + options: {}, + headers: {}, + release_date: "2026-01-01", + variants: {}, +} + +const provider: Provider.Info = { + id: providerID, + name: "Simulation", + source: "custom", + env: [], + options: {}, + models: { [modelID]: model }, +} + +function text(script: LLMScript) { + return script.steps[0]?.flatMap((item) => (item.type === "text" || item.type === "thinking" ? [item.content] : [])) + .join("") ?? "" +} + +function error(script: LLMScript) { + return script.steps[0]?.find((item) => item.type === "error") +} + +function stream(script: LLMScript) { + return new ReadableStream({ + start(controller) { + controller.enqueue({ type: "stream-start", warnings: [] }) + let index = 0 + for (const item of script.steps[0] ?? []) { + index++ + if (item.type === "error") { + controller.enqueue({ type: "error", error: new Error(item.message) }) + controller.close() + return + } + const id = `simulation-${item.type}-${index}` + if (item.type === "thinking") { + controller.enqueue({ type: "reasoning-start", id }) + controller.enqueue({ type: "reasoning-delta", id, delta: item.content }) + controller.enqueue({ type: "reasoning-end", id }) + continue + } + controller.enqueue({ type: "text-start", id }) + controller.enqueue({ type: "text-delta", id, delta: item.content }) + controller.enqueue({ type: "text-end", id }) + } + controller.enqueue({ type: "finish", finishReason: finishReason(script), usage: usage(script) }) + controller.close() + }, + }) +} + +function usage(script: LLMScript) { + return { + inputTokens: { + total: script.usage?.inputTokens ?? 0, + noCache: script.usage?.inputTokens ?? 0, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { + total: script.usage?.outputTokens ?? text(script).length, + text: script.usage?.outputTokens ?? text(script).length, + reasoning: undefined, + }, + raw: script.usage, + } +} + +function finishReason(script: LLMScript): LanguageModelV3FinishReason { + return { unified: script.finish === "unknown" ? "other" : (script.finish ?? "stop"), raw: script.finish } +} + +function language(simulation: Simulation.Interface): LanguageModelV3 { + return { + specificationVersion: "v3", + provider: "simulation", + modelId: modelID, + supportedUrls: {}, + async doGenerate(_options: LanguageModelV3CallOptions) { + const script = await Effect.runPromise(simulation.nextLLM()) + const err = error(script) + if (err?.type === "error") throw new Error(err.message) + return { + content: [{ type: "text", text: text(script) }], + finishReason: finishReason(script), + usage: usage(script), + warnings: [], + } + }, + async doStream(_options: LanguageModelV3CallOptions) { + const script = await Effect.runPromise(simulation.nextLLM()) + return { stream: stream(script) } + }, + } +} + +export const layer = Layer.effect( + Provider.Service, + Effect.gen(function* () { + const simulation = yield* Simulation.Service + const lang = language(simulation) + return Provider.Service.of({ + list: () => Effect.succeed({ [providerID]: provider }), + getProvider: () => Effect.succeed(provider), + getModel: () => Effect.succeed(model), + getLanguage: () => Effect.succeed(lang), + closest: () => Effect.succeed({ providerID, modelID }), + getSmallModel: () => Effect.succeed(model), + defaultModel: () => Effect.succeed({ providerID, modelID }), + }) + }), +) + +export * as SimulationProvider from "./provider" diff --git a/packages/opencode/src/testing/simulation/service.ts b/packages/opencode/src/testing/simulation/service.ts index dff1e0bb2..222c1d333 100644 --- a/packages/opencode/src/testing/simulation/service.ts +++ b/packages/opencode/src/testing/simulation/service.ts @@ -38,20 +38,56 @@ export const NetworkRegisterInput = Schema.Union([ }), ]) +export const LLMScriptAction = Schema.Union([ + Schema.Struct({ type: Schema.Literal("text"), content: Schema.String }), + Schema.Struct({ type: Schema.Literal("thinking"), content: Schema.String }), + Schema.Struct({ type: Schema.Literal("error"), message: Schema.String }), +]) + +export type LLMScriptAction = typeof LLMScriptAction.Type + +export const LLMScript = Schema.Struct({ + steps: Schema.Array(Schema.Array(LLMScriptAction)), + usage: Schema.optional( + Schema.Struct({ + inputTokens: Schema.Number, + outputTokens: Schema.Number, + totalTokens: Schema.Number, + }), + ), + finish: Schema.optional(Schema.Literals(["stop", "tool-calls", "error", "length", "unknown"])), +}) + +export type LLMScript = typeof LLMScript.Type + +export const LLMEnqueueInput = Schema.Struct({ + scripts: Schema.Array(LLMScript), +}) + type FilePath = string interface State { readonly files: readonly FilePath[] readonly networkRegistrations: readonly string[] + readonly llmScripts: readonly LLMScript[] + readonly consumedLLMScripts: number } +export class SimulationLLMError extends Schema.TaggedErrorClass()("SimulationLLMError", { + message: Schema.String, +}) {} + export interface Interface { readonly reset: () => Effect.Effect readonly seedFilesystem: (input: typeof FilesystemSeedInput.Type) => Effect.Effect<{ files: string[] }, unknown> readonly registerNetwork: (input: typeof NetworkRegisterInput.Type) => Effect.Effect<{ registered: string }, unknown> + readonly enqueueLLM: (input: typeof LLMEnqueueInput.Type) => Effect.Effect<{ queued: number }> + readonly nextLLM: () => Effect.Effect readonly snapshot: () => Effect.Effect<{ files: readonly string[] networkRegistrations: readonly string[] + llmQueued: number + llmConsumed: number network: SimulationNetwork.Snapshot }> } @@ -72,11 +108,11 @@ export const layer = Layer.effect( Effect.gen(function* () { const fs = yield* AppFileSystem.Service const network = yield* SimulationNetwork.Service - const state = yield* Ref.make({ files: [], networkRegistrations: [] }) + const state = yield* Ref.make({ files: [], networkRegistrations: [], llmScripts: [], consumedLLMScripts: 0 }) const reset = Effect.fn("Simulation.reset")(function* () { yield* network.reset() - yield* Ref.set(state, { files: [], networkRegistrations: [] }) + yield* Ref.set(state, { files: [], networkRegistrations: [], llmScripts: [], consumedLLMScripts: 0 }) }) const seedFilesystem = Effect.fn("Simulation.seedFilesystem")(function* (input: typeof FilesystemSeedInput.Type) { @@ -112,12 +148,31 @@ export const layer = Layer.effect( return { registered: input.url } }) - const snapshot = Effect.fn("Simulation.snapshot")(function* () { - const current = yield* Ref.get(state) - return { ...current, network: yield* network.snapshot() } + const enqueueLLM = Effect.fn("Simulation.enqueueLLM")(function* (input: typeof LLMEnqueueInput.Type) { + yield* Ref.update(state, (current) => ({ ...current, llmScripts: [...current.llmScripts, ...input.scripts] })) + return { queued: input.scripts.length } }) - return Service.of({ reset, seedFilesystem, registerNetwork, snapshot }) + const nextLLM = Effect.fn("Simulation.nextLLM")(function* () { + const current = yield* Ref.get(state) + const [script, ...rest] = current.llmScripts + if (!script) return yield* new SimulationLLMError({ message: "No LLM script queued" }) + yield* Ref.set(state, { ...current, llmScripts: rest, consumedLLMScripts: current.consumedLLMScripts + 1 }) + return script + }) + + const snapshot = Effect.fn("Simulation.snapshot")(function* () { + const current = yield* Ref.get(state) + return { + files: current.files, + networkRegistrations: current.networkRegistrations, + llmQueued: current.llmScripts.length, + llmConsumed: current.consumedLLMScripts, + network: yield* network.snapshot(), + } + }) + + return Service.of({ reset, seedFilesystem, registerNetwork, enqueueLLM, nextLLM, snapshot }) }), ) diff --git a/packages/opencode/test/testing/simulation/service.test.ts b/packages/opencode/test/testing/simulation/service.test.ts index 573affb6f..e1bb20486 100644 --- a/packages/opencode/test/testing/simulation/service.test.ts +++ b/packages/opencode/test/testing/simulation/service.test.ts @@ -2,15 +2,18 @@ import { describe, expect } from "bun:test" import { AppFileSystem } from "@opencode-ai/core/filesystem" import { Effect, Layer } from "effect" import { HttpClient, HttpClientRequest } from "effect/unstable/http" +import { Provider } from "../../../src/provider/provider" import { SimulationFileSystem } from "../../../src/testing/simulation/filesystem" import { SimulationNetwork } from "../../../src/testing/simulation/network" +import { SimulationProvider } from "../../../src/testing/simulation/provider" import { Simulation } from "../../../src/testing/simulation/service" import { testEffect } from "../../lib/effect" const fsLayer = SimulationFileSystem.layer({ root: "/opencode" }) const networkLayer = SimulationNetwork.layer({ allowLoopback: false }) const simulationLayer = Simulation.layer.pipe(Layer.provide(fsLayer), Layer.provide(networkLayer)) -const it = testEffect(Layer.mergeAll(fsLayer, networkLayer, simulationLayer)) +const providerLayer = SimulationProvider.layer.pipe(Layer.provide(simulationLayer)) +const it = testEffect(Layer.mergeAll(fsLayer, networkLayer, simulationLayer, providerLayer)) describe("Simulation", () => { it.effect("seeds files into the simulated filesystem", () => @@ -64,4 +67,85 @@ describe("Simulation", () => { expect(exit._tag).toBe("Failure") }), ) + + it.effect("queues and consumes LLM scripts", () => + Effect.gen(function* () { + const simulation = yield* Simulation.Service + + expect( + yield* simulation.enqueueLLM({ + scripts: [{ steps: [[{ type: "text", content: "hello" }]], finish: "stop" }], + }), + ).toEqual({ queued: 1 }) + expect((yield* simulation.snapshot()).llmQueued).toBe(1) + expect(yield* simulation.nextLLM()).toEqual({ steps: [[{ type: "text", content: "hello" }]], finish: "stop" }) + expect((yield* simulation.snapshot()).llmConsumed).toBe(1) + }), + ) + + it.effect("simulation provider consumes queued text scripts", () => + Effect.gen(function* () { + const simulation = yield* Simulation.Service + const provider = yield* Provider.Service + const model = yield* provider.defaultModel().pipe(Effect.flatMap((item) => provider.getModel(item.providerID, item.modelID))) + const language = yield* provider.getLanguage(model) + + yield* simulation.enqueueLLM({ scripts: [{ steps: [[{ type: "text", content: "assistant text" }]] }] }) + + const result = yield* Effect.promise(() => language.doGenerate({ prompt: [], abortSignal: undefined })) + expect(result.content).toEqual([{ type: "text", text: "assistant text" }]) + expect((yield* simulation.snapshot()).llmConsumed).toBe(1) + }), + ) + + it.effect("simulation provider streams queued script actions", () => + Effect.gen(function* () { + const simulation = yield* Simulation.Service + const provider = yield* Provider.Service + const model = yield* provider.defaultModel().pipe(Effect.flatMap((item) => provider.getModel(item.providerID, item.modelID))) + const language = yield* provider.getLanguage(model) + + yield* simulation.enqueueLLM({ + scripts: [ + { + steps: [ + [ + { type: "thinking", content: "thinking" }, + { type: "text", content: "answer" }, + ], + ], + }, + ], + }) + + const result = yield* Effect.promise(() => language.doStream({ prompt: [], abortSignal: undefined })) + const reader = result.stream.getReader() + const parts: unknown[] = [] + while (true) { + const next = yield* Effect.promise(() => reader.read()) + if (next.done) break + parts.push(next.value) + } + + expect(parts).toEqual([ + { type: "stream-start", warnings: [] }, + { type: "reasoning-start", id: "simulation-thinking-1" }, + { type: "reasoning-delta", id: "simulation-thinking-1", delta: "thinking" }, + { type: "reasoning-end", id: "simulation-thinking-1" }, + { type: "text-start", id: "simulation-text-2" }, + { type: "text-delta", id: "simulation-text-2", delta: "answer" }, + { type: "text-end", id: "simulation-text-2" }, + { + type: "finish", + finishReason: { unified: "stop", raw: undefined }, + usage: { + inputTokens: { total: 0, noCache: 0, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: "thinkinganswer".length, text: "thinkinganswer".length, reasoning: undefined }, + raw: undefined, + }, + }, + ]) + expect((yield* simulation.snapshot()).llmConsumed).toBe(1) + }), + ) })