fix(session): include image source paths in model context

This commit is contained in:
Ryan Vogel
2026-05-19 14:32:27 +00:00
parent 2932a7a35d
commit 8e726761e2
2 changed files with 126 additions and 0 deletions
@@ -627,6 +627,26 @@ function providerMeta(metadata: Record<string, any> | undefined) {
return Object.keys(rest).length > 0 ? rest : undefined
}
function fileSourceContext(part: FilePart) {
const source = part.source
if (!source) return
if (source.type === "resource") return
const filepath = source.path.trim()
if (!filepath) return
if (filepath === "clipboard" && part.filename === "clipboard") return
const name = part.filename?.trim()
const reference = source.text.value.trim()
return [
`Attached ${part.mime.startsWith("image/") ? "image" : "file"}${name ? `: ${name}` : ""}`,
`Source path: ${filepath}`,
reference ? `Prompt reference: ${reference}` : undefined,
]
.filter((item): item is string => !!item)
.join("\n")
}
export const toModelMessagesEffect = Effect.fnUntraced(function* (
input: WithParts[],
model: Provider.Model,
@@ -707,6 +727,13 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* (
})
// text/plain and directory files are converted into text parts, ignore them
if (part.type === "file" && part.mime !== "text/plain" && part.mime !== "application/x-directory") {
const context = fileSourceContext(part)
if (context) {
userMessage.parts.push({
type: "text",
text: context,
})
}
if (options?.stripMedia && isMedia(part.mime)) {
userMessage.parts.push({
type: "text",
@@ -316,6 +316,105 @@ describe("session.message-v2.toModelMessage", () => {
])
})
test("includes source paths for user media attachments", async () => {
const messageID = "m-user-source"
const result = await MessageV2.toModelMessages(
[
{
info: userInfo(messageID),
parts: [
{
...basePart(messageID, "p1-source"),
type: "text",
text: "copy this image",
},
{
...basePart(messageID, "p2-source"),
type: "file",
mime: "image/png",
filename: "screenshot.png",
url: "data:image/png;base64,AAA",
source: {
type: "file",
path: "/repo/assets/screenshot.png",
text: {
value: "[Image 1]",
start: 16,
end: 25,
},
},
},
] as MessageV2.Part[],
},
],
model,
)
expect(result).toMatchObject([
{
role: "user",
content: [
{ type: "text", text: "copy this image" },
{
type: "text",
text: "Attached image: screenshot.png\nSource path: /repo/assets/screenshot.png\nPrompt reference: [Image 1]",
},
{
type: "file",
mediaType: "image/png",
filename: "screenshot.png",
},
],
},
])
})
test("does not expose clipboard placeholder as a source path", async () => {
const messageID = "m-user-clipboard"
expect(
await MessageV2.toModelMessages(
[
{
info: userInfo(messageID),
parts: [
{
...basePart(messageID, "p1-clipboard"),
type: "file",
mime: "image/png",
filename: "clipboard",
url: "data:image/png;base64,AAA",
source: {
type: "file",
path: "clipboard",
text: {
value: "[Image 1]",
start: 0,
end: 9,
},
},
},
] as MessageV2.Part[],
},
],
model,
),
).toStrictEqual([
{
role: "user",
content: [
{
type: "file",
mediaType: "image/png",
filename: "clipboard",
data: "data:image/png;base64,AAA",
},
],
},
])
})
test("converts assistant tool completion into tool-call + tool-result messages with attachments", async () => {
const userID = "m-user"
const assistantID = "m-assistant"