support token caching for anthropic via openrouter

This commit is contained in:
Dax Raad
2025-06-19 10:32:14 -04:00
parent 4b132656df
commit f1a2b2eba4
2 changed files with 32 additions and 21 deletions

View File

@@ -0,0 +1,24 @@
import type { CoreMessage } from "ai"
export namespace ProviderTransform {
export function message(
msg: CoreMessage,
index: number,
providerID: string,
modelID: string,
) {
if (
(providerID === "anthropic" || modelID.includes("anthropic")) &&
index < 4
) {
msg.providerOptions = {
...msg.providerOptions,
anthropic: {
cacheControl: { type: "ephemeral" },
},
}
}
return msg
}
}

View File

@@ -32,6 +32,7 @@ import { Flag } from "../flag/flag"
import type { ModelsDev } from "../provider/models" import type { ModelsDev } from "../provider/models"
import { Installation } from "../installation" import { Installation } from "../installation"
import { Config } from "../config/config" import { Config } from "../config/config"
import { ProviderTransform } from "../provider/transform"
export namespace Session { export namespace Session {
const log = Log.create({ service: "session" }) const log = Log.create({ service: "session" })
@@ -266,15 +267,6 @@ export namespace Session {
(x): CoreMessage => ({ (x): CoreMessage => ({
role: "system", role: "system",
content: x, content: x,
providerOptions: {
...(input.providerID === "anthropic"
? {
anthropic: {
cacheControl: { type: "ephemeral" },
},
}
: {}),
},
}), }),
), ),
...convertToCoreMessages([ ...convertToCoreMessages([
@@ -284,7 +276,9 @@ export namespace Session {
parts: toParts(input.parts), parts: toParts(input.parts),
}, },
]), ]),
], ].map((msg, i) =>
ProviderTransform.message(msg, i, input.providerID, input.modelID),
),
model: model.language, model: model.language,
}) })
.then((result) => { .then((result) => {
@@ -515,24 +509,17 @@ export namespace Session {
maxSteps: 1000, maxSteps: 1000,
messages: [ messages: [
...system.map( ...system.map(
(x, index): CoreMessage => ({ (x): CoreMessage => ({
role: "system", role: "system",
content: x, content: x,
providerOptions: {
...(input.providerID === "anthropic" && index < 4
? {
anthropic: {
cacheControl: { type: "ephemeral" },
},
}
: {}),
},
}), }),
), ),
...convertToCoreMessages( ...convertToCoreMessages(
msgs.map(toUIMessage).filter((x) => x.parts.length > 0), msgs.map(toUIMessage).filter((x) => x.parts.length > 0),
), ),
], ].map((msg, i) =>
ProviderTransform.message(msg, i, input.providerID, input.modelID),
),
temperature: model.info.temperature ? 0 : undefined, temperature: model.info.temperature ? 0 : undefined,
tools: { tools: {
...tools, ...tools,