fix(core): isolate image normalization (#31165)
This commit is contained in:
Kit Langton
GitHub
parent
12acb9a59a
commit
10d1e04e9b
72
packages/core/src/image.ts
Normal file
72
packages/core/src/image.ts
Normal file
@ -0,0 +1,72 @@
|
||||
export * as Image from "./image"
|
||||
|
||||
import { Context, Effect, Layer, Schema } from "effect"
|
||||
import { Config } from "./config"
|
||||
import { FileSystem } from "./filesystem"
|
||||
|
||||
export class ResizerUnavailableError extends Schema.TaggedErrorClass<ResizerUnavailableError>()(
|
||||
"Image.ResizerUnavailableError",
|
||||
{},
|
||||
) {}
|
||||
|
||||
export class DecodeError extends Schema.TaggedErrorClass<DecodeError>()("Image.DecodeError", {
|
||||
resource: Schema.String,
|
||||
}) {
|
||||
override get message() {
|
||||
return `Image could not be decoded: ${this.resource}`
|
||||
}
|
||||
}
|
||||
|
||||
export class SizeError extends Schema.TaggedErrorClass<SizeError>()("Image.SizeError", {
|
||||
resource: Schema.String,
|
||||
width: Schema.Number,
|
||||
height: Schema.Number,
|
||||
bytes: Schema.Number,
|
||||
maxWidth: Schema.Number,
|
||||
maxHeight: Schema.Number,
|
||||
maxBytes: Schema.Number,
|
||||
}) {
|
||||
override get message() {
|
||||
return `Image ${this.resource} is ${this.width}x${this.height} with base64 size ${this.bytes}, exceeding configured limits ${this.maxWidth}x${this.maxHeight}/${this.maxBytes} bytes`
|
||||
}
|
||||
}
|
||||
|
||||
export interface Interface {
|
||||
readonly normalize: (
|
||||
resource: string,
|
||||
content: FileSystem.BinaryContent,
|
||||
) => Effect.Effect<FileSystem.BinaryContent, ResizerUnavailableError | DecodeError | SizeError>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/Image") {}
|
||||
|
||||
export const layer = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const config = yield* Config.Service
|
||||
const loadAdapter = yield* Effect.cached(
|
||||
Effect.tryPromise({
|
||||
try: () => import("./image/photon"),
|
||||
catch: () => new ResizerUnavailableError(),
|
||||
}).pipe(Effect.flatMap((adapter) => adapter.make)),
|
||||
)
|
||||
const normalize = Effect.fn("Image.normalize")(function* (resource: string, content: FileSystem.BinaryContent) {
|
||||
const image = Object.assign(
|
||||
{},
|
||||
...(yield* config.entries()).flatMap((entry) =>
|
||||
entry.type === "document" && entry.info.attachments?.image ? [entry.info.attachments.image] : [],
|
||||
),
|
||||
)
|
||||
const normalize = yield* loadAdapter
|
||||
return yield* normalize(resource, content, {
|
||||
autoResize: image.auto_resize ?? true,
|
||||
maxWidth: image.max_width ?? 2_000,
|
||||
maxHeight: image.max_height ?? 2_000,
|
||||
maxBase64Bytes: image.max_base64_bytes ?? 5 * 1024 * 1024,
|
||||
})
|
||||
})
|
||||
return Service.of({ normalize })
|
||||
}),
|
||||
)
|
||||
|
||||
export const locationLayer = layer.pipe(Layer.provide(Config.locationLayer))
|
||||
94
packages/core/src/image/photon.ts
Normal file
94
packages/core/src/image/photon.ts
Normal file
@ -0,0 +1,94 @@
|
||||
// @ts-ignore Bun's static file import is embedded by `bun build --compile`; some consumers also declare *.wasm.
|
||||
import photonWasm from "@silvia-odwyer/photon-node/photon_rs_bg.wasm" with { type: "file" }
|
||||
import { Effect } from "effect"
|
||||
import path from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
import { FileSystem } from "../filesystem"
|
||||
import { DecodeError, ResizerUnavailableError, SizeError } from "../image"
|
||||
|
||||
const JPEG_QUALITIES = [80, 85, 70, 55, 40]
|
||||
|
||||
export const make = Effect.gen(function* () {
|
||||
;(globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }).__OPENCODE_PHOTON_WASM_PATH =
|
||||
path.isAbsolute(photonWasm) ? photonWasm : fileURLToPath(new URL(photonWasm, import.meta.url))
|
||||
const loadPhoton = yield* Effect.cached(
|
||||
Effect.tryPromise({
|
||||
try: () => import("@silvia-odwyer/photon-node"),
|
||||
catch: () => new ResizerUnavailableError(),
|
||||
}),
|
||||
)
|
||||
return Effect.fn("Image.Photon.normalize")(function* (
|
||||
resource: string,
|
||||
content: FileSystem.BinaryContent,
|
||||
limits: {
|
||||
readonly autoResize: boolean
|
||||
readonly maxWidth: number
|
||||
readonly maxHeight: number
|
||||
readonly maxBase64Bytes: number
|
||||
},
|
||||
) {
|
||||
const photon = yield* loadPhoton
|
||||
const decoded = yield* Effect.try({
|
||||
try: () => photon.PhotonImage.new_from_byteslice(Buffer.from(content.content, "base64")),
|
||||
catch: () => new DecodeError({ resource }),
|
||||
})
|
||||
try {
|
||||
const width = decoded.get_width()
|
||||
const height = decoded.get_height()
|
||||
const bytes = Buffer.byteLength(content.content, "utf-8")
|
||||
if (width <= limits.maxWidth && height <= limits.maxHeight && bytes <= limits.maxBase64Bytes) return content
|
||||
if (!limits.autoResize)
|
||||
return yield* new SizeError({
|
||||
resource,
|
||||
width,
|
||||
height,
|
||||
bytes,
|
||||
maxWidth: limits.maxWidth,
|
||||
maxHeight: limits.maxHeight,
|
||||
maxBytes: limits.maxBase64Bytes,
|
||||
})
|
||||
const scale = Math.min(1, limits.maxWidth / width, limits.maxHeight / height)
|
||||
const sizes = Array.from({ length: 32 }).reduce<Array<{ width: number; height: number }>>((acc) => {
|
||||
const previous = acc.at(-1) ?? {
|
||||
width: Math.max(1, Math.round(width * scale)),
|
||||
height: Math.max(1, Math.round(height * scale)),
|
||||
}
|
||||
const next =
|
||||
acc.length === 0
|
||||
? previous
|
||||
: {
|
||||
width: previous.width === 1 ? 1 : Math.max(1, Math.floor(previous.width * 0.75)),
|
||||
height: previous.height === 1 ? 1 : Math.max(1, Math.floor(previous.height * 0.75)),
|
||||
}
|
||||
return acc.some((item) => item.width === next.width && item.height === next.height) ? acc : [...acc, next]
|
||||
}, [])
|
||||
for (const size of sizes) {
|
||||
const resized = photon.resize(decoded, size.width, size.height, photon.SamplingFilter.Lanczos3)
|
||||
try {
|
||||
const encoders: Array<readonly [mime: string, encode: () => Uint8Array]> = [
|
||||
["image/png", () => resized.get_bytes()],
|
||||
...JPEG_QUALITIES.map((quality) => ["image/jpeg", () => resized.get_bytes_jpeg(quality)] as const),
|
||||
]
|
||||
for (const [mime, encode] of encoders) {
|
||||
const candidate = Buffer.from(encode()).toString("base64")
|
||||
if (Buffer.byteLength(candidate, "utf-8") <= limits.maxBase64Bytes)
|
||||
return new FileSystem.BinaryContent({ type: "binary", content: candidate, encoding: "base64", mime })
|
||||
}
|
||||
} finally {
|
||||
resized.free()
|
||||
}
|
||||
}
|
||||
return yield* new SizeError({
|
||||
resource,
|
||||
width,
|
||||
height,
|
||||
bytes,
|
||||
maxWidth: limits.maxWidth,
|
||||
maxHeight: limits.maxHeight,
|
||||
maxBytes: limits.maxBase64Bytes,
|
||||
})
|
||||
} finally {
|
||||
decoded.free()
|
||||
}
|
||||
})
|
||||
})
|
||||
@ -28,6 +28,7 @@ import { Pty } from "./pty"
|
||||
import { SkillV2 } from "./skill"
|
||||
import { SkillGuidance } from "./skill/guidance"
|
||||
import { BuiltInTools } from "./tool/builtins"
|
||||
import { Image } from "./image"
|
||||
import { ToolRegistry } from "./tool/registry"
|
||||
import { ApplicationTools } from "./tool/application-tools"
|
||||
import { ToolOutputStore } from "./tool-output-store"
|
||||
@ -71,6 +72,7 @@ export class LocationServiceMap extends LayerMap.Service<LocationServiceMap>()("
|
||||
Layer.provide(base),
|
||||
)
|
||||
const services = Layer.mergeAll(base, resources, permissionsAndTools)
|
||||
const image = Image.layer.pipe(Layer.provide(services))
|
||||
const mutation = FileMutation.locationLayer.pipe(Layer.provide(services))
|
||||
const searches = LocationSearch.layer.pipe(Layer.provide(Ripgrep.layer), Layer.provide(services))
|
||||
const skillGuidance = SkillGuidance.locationLayer.pipe(Layer.provide(services))
|
||||
@ -83,6 +85,7 @@ export class LocationServiceMap extends LayerMap.Service<LocationServiceMap>()("
|
||||
Layer.provide(resources),
|
||||
Layer.provide(todos),
|
||||
Layer.provide(questions),
|
||||
Layer.provide(image),
|
||||
)
|
||||
const model = SessionRunnerModel.locationLayer.pipe(Layer.provide(services))
|
||||
const runner = SessionRunnerLLM.defaultLayer.pipe(
|
||||
@ -90,9 +93,18 @@ export class LocationServiceMap extends LayerMap.Service<LocationServiceMap>()("
|
||||
Layer.provide(model),
|
||||
Layer.provide(skillGuidance),
|
||||
)
|
||||
return Layer.mergeAll(services, mutation, searches, resources, todos, questions, model, runner, builtInTools).pipe(
|
||||
Layer.fresh,
|
||||
)
|
||||
return Layer.mergeAll(
|
||||
services,
|
||||
image,
|
||||
mutation,
|
||||
searches,
|
||||
resources,
|
||||
todos,
|
||||
questions,
|
||||
model,
|
||||
runner,
|
||||
builtInTools,
|
||||
).pipe(Layer.fresh)
|
||||
},
|
||||
idleTimeToLive: "60 minutes",
|
||||
dependencies: [
|
||||
|
||||
@ -1,47 +1,15 @@
|
||||
export * as ReadTool from "./read"
|
||||
|
||||
import { ToolFailure } from "@opencode-ai/llm"
|
||||
// @ts-ignore Bun's static file import is embedded by `bun build --compile`; some consumers also declare *.wasm.
|
||||
import photonWasm from "@silvia-odwyer/photon-node/photon_rs_bg.wasm" with { type: "file" }
|
||||
import { Effect, Layer, Schema } from "effect"
|
||||
import path from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
import { Config } from "../config"
|
||||
import { FileSystem } from "../filesystem"
|
||||
import { Image } from "../image"
|
||||
import { PermissionV2 } from "../permission"
|
||||
import { Tool } from "./tool"
|
||||
import { Tools } from "./tools"
|
||||
|
||||
export const name = "read"
|
||||
const SUPPORTED_IMAGE_MIMES = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"])
|
||||
const MAX_IMAGE_BASE64_BYTES = 5 * 1024 * 1024
|
||||
const MAX_IMAGE_WIDTH = 2_000
|
||||
const MAX_IMAGE_HEIGHT = 2_000
|
||||
const JPEG_QUALITIES = [80, 85, 70, 55, 40]
|
||||
|
||||
class ImageDecodeError extends Error {
|
||||
constructor(readonly resource: string) {
|
||||
super(`Image could not be decoded: ${resource}`)
|
||||
this.name = "ImageDecodeError"
|
||||
}
|
||||
}
|
||||
|
||||
class ImageSizeError extends Error {
|
||||
constructor(
|
||||
readonly resource: string,
|
||||
readonly width: number,
|
||||
readonly height: number,
|
||||
readonly bytes: number,
|
||||
readonly maxWidth: number,
|
||||
readonly maxHeight: number,
|
||||
readonly maxBytes: number,
|
||||
) {
|
||||
super(
|
||||
`Image ${resource} is ${width}x${height} with base64 size ${bytes}, exceeding configured limits ${maxWidth}x${maxHeight}/${maxBytes} bytes`,
|
||||
)
|
||||
this.name = "ImageSizeError"
|
||||
}
|
||||
}
|
||||
const LocationInput = Schema.Struct({
|
||||
...FileSystem.ReadInput.fields,
|
||||
offset: FileSystem.ListPageInput.fields.offset.annotate({
|
||||
@ -58,14 +26,8 @@ export const layer = Layer.effectDiscard(
|
||||
Effect.gen(function* () {
|
||||
const tools = yield* Tools.Service
|
||||
const filesystem = yield* FileSystem.Service
|
||||
const config = yield* Config.Service
|
||||
const image = yield* Image.Service
|
||||
const permission = yield* PermissionV2.Service
|
||||
const loadPhoton = yield* Effect.cached(
|
||||
Effect.sync(() => {
|
||||
;(globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }).__OPENCODE_PHOTON_WASM_PATH =
|
||||
path.isAbsolute(photonWasm) ? photonWasm : fileURLToPath(new URL(photonWasm, import.meta.url))
|
||||
}).pipe(Effect.andThen(() => Effect.promise(() => import("@silvia-odwyer/photon-node")))),
|
||||
)
|
||||
|
||||
yield* tools
|
||||
.register({
|
||||
@ -98,95 +60,9 @@ export const layer = Layer.effectDiscard(
|
||||
limit: input.limit,
|
||||
})
|
||||
if (content.type === "binary" && SUPPORTED_IMAGE_MIMES.has(content.mime)) {
|
||||
const mime = content.mime
|
||||
const base64 = content.content
|
||||
const image = Object.assign(
|
||||
{},
|
||||
...(yield* config.entries()).flatMap((entry) =>
|
||||
entry.type === "document" && entry.info.attachments?.image ? [entry.info.attachments.image] : [],
|
||||
),
|
||||
)
|
||||
const limits = {
|
||||
autoResize: image.auto_resize ?? true,
|
||||
maxWidth: image.max_width ?? MAX_IMAGE_WIDTH,
|
||||
maxHeight: image.max_height ?? MAX_IMAGE_HEIGHT,
|
||||
maxBase64Bytes: image.max_base64_bytes ?? MAX_IMAGE_BASE64_BYTES,
|
||||
}
|
||||
const photon = yield* loadPhoton
|
||||
const decoded = yield* Effect.try({
|
||||
try: () => photon.PhotonImage.new_from_byteslice(Buffer.from(base64, "base64")),
|
||||
catch: () => new ImageDecodeError(resolved.resource),
|
||||
})
|
||||
try {
|
||||
const width = decoded.get_width()
|
||||
const height = decoded.get_height()
|
||||
const bytes = Buffer.byteLength(base64, "utf-8")
|
||||
if (width <= limits.maxWidth && height <= limits.maxHeight && bytes <= limits.maxBase64Bytes)
|
||||
return new FileSystem.BinaryContent({ type: "binary", content: base64, encoding: "base64", mime })
|
||||
if (!limits.autoResize)
|
||||
return yield* Effect.fail(
|
||||
new ImageSizeError(
|
||||
resolved.resource,
|
||||
width,
|
||||
height,
|
||||
bytes,
|
||||
limits.maxWidth,
|
||||
limits.maxHeight,
|
||||
limits.maxBase64Bytes,
|
||||
),
|
||||
)
|
||||
const scale = Math.min(1, limits.maxWidth / width, limits.maxHeight / height)
|
||||
const sizes = Array.from({ length: 32 }).reduce<Array<{ width: number; height: number }>>((acc) => {
|
||||
const previous = acc.at(-1) ?? {
|
||||
width: Math.max(1, Math.round(width * scale)),
|
||||
height: Math.max(1, Math.round(height * scale)),
|
||||
}
|
||||
const next =
|
||||
acc.length === 0
|
||||
? previous
|
||||
: {
|
||||
width: previous.width === 1 ? 1 : Math.max(1, Math.floor(previous.width * 0.75)),
|
||||
height: previous.height === 1 ? 1 : Math.max(1, Math.floor(previous.height * 0.75)),
|
||||
}
|
||||
return acc.some((item) => item.width === next.width && item.height === next.height)
|
||||
? acc
|
||||
: [...acc, next]
|
||||
}, [])
|
||||
for (const size of sizes) {
|
||||
const resized = photon.resize(decoded, size.width, size.height, photon.SamplingFilter.Lanczos3)
|
||||
try {
|
||||
const candidate = [
|
||||
{ content: Buffer.from(resized.get_bytes()).toString("base64"), mime: "image/png" },
|
||||
...JPEG_QUALITIES.map((quality) => ({
|
||||
content: Buffer.from(resized.get_bytes_jpeg(quality)).toString("base64"),
|
||||
mime: "image/jpeg",
|
||||
})),
|
||||
].find((item) => Buffer.byteLength(item.content, "utf-8") <= limits.maxBase64Bytes)
|
||||
if (candidate)
|
||||
return new FileSystem.BinaryContent({
|
||||
type: "binary",
|
||||
content: candidate.content,
|
||||
encoding: "base64",
|
||||
mime: candidate.mime,
|
||||
})
|
||||
} finally {
|
||||
resized.free()
|
||||
}
|
||||
}
|
||||
return yield* Effect.fail(
|
||||
new ImageSizeError(
|
||||
resolved.resource,
|
||||
width,
|
||||
height,
|
||||
bytes,
|
||||
limits.maxWidth,
|
||||
limits.maxHeight,
|
||||
limits.maxBase64Bytes,
|
||||
),
|
||||
)
|
||||
} finally {
|
||||
decoded.free()
|
||||
}
|
||||
return yield* image
|
||||
.normalize(resolved.resource, content)
|
||||
.pipe(Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(content)))
|
||||
}
|
||||
if (content.type === "binary")
|
||||
return yield* Effect.fail(new FileSystem.BinaryFileError(resolved.resource))
|
||||
@ -196,8 +72,8 @@ export const layer = Layer.effectDiscard(
|
||||
const message =
|
||||
error instanceof FileSystem.BinaryFileError ||
|
||||
error instanceof FileSystem.MediaIngestLimitError ||
|
||||
error instanceof ImageDecodeError ||
|
||||
error instanceof ImageSizeError
|
||||
error instanceof Image.DecodeError ||
|
||||
error instanceof Image.SizeError
|
||||
? error.message
|
||||
: `Unable to read ${input.path}`
|
||||
return new ToolFailure({ message })
|
||||
|
||||
@ -3,6 +3,7 @@ import { Effect, Exit, Layer } from "effect"
|
||||
import { Config } from "@opencode-ai/core/config"
|
||||
import { ConfigAttachments } from "@opencode-ai/core/config/attachments"
|
||||
import { FileSystem } from "@opencode-ai/core/filesystem"
|
||||
import { Image } from "@opencode-ai/core/image"
|
||||
import { PermissionV2 } from "@opencode-ai/core/permission"
|
||||
import { SessionV2 } from "@opencode-ai/core/session"
|
||||
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
||||
@ -75,13 +76,29 @@ const permission = Layer.succeed(
|
||||
)
|
||||
const registry = ToolRegistry.defaultLayer.pipe(Layer.provide(permission))
|
||||
const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () => Effect.succeed(configEntries) }))
|
||||
const image = Image.layer.pipe(Layer.provide(config))
|
||||
const unavailableImage = Layer.succeed(
|
||||
Image.Service,
|
||||
Image.Service.of({ normalize: () => Effect.fail(new Image.ResizerUnavailableError()) }),
|
||||
)
|
||||
const read = ReadTool.layer.pipe(
|
||||
Layer.provide(registry),
|
||||
Layer.provide(filesystem),
|
||||
Layer.provide(permission),
|
||||
Layer.provide(config),
|
||||
Layer.provide(image),
|
||||
)
|
||||
const it = testEffect(Layer.mergeAll(registry, filesystem, permission, config, image, read))
|
||||
const unavailableRead = ReadTool.layer.pipe(
|
||||
Layer.provide(registry),
|
||||
Layer.provide(filesystem),
|
||||
Layer.provide(permission),
|
||||
Layer.provide(config),
|
||||
Layer.provide(unavailableImage),
|
||||
)
|
||||
const itWithoutResizer = testEffect(
|
||||
Layer.mergeAll(registry, filesystem, permission, config, unavailableImage, unavailableRead),
|
||||
)
|
||||
const it = testEffect(Layer.mergeAll(registry, filesystem, permission, config, read))
|
||||
const sessionID = SessionV2.ID.make("ses_read_tool_test")
|
||||
|
||||
describe("ReadTool", () => {
|
||||
@ -188,6 +205,30 @@ describe("ReadTool", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
itWithoutResizer.effect("returns the original image when the resizer is unavailable", () =>
|
||||
Effect.gen(function* () {
|
||||
const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
|
||||
readResult = new FileSystem.BinaryContent({
|
||||
type: "binary",
|
||||
content: png,
|
||||
encoding: "base64",
|
||||
mime: "image/png",
|
||||
})
|
||||
const registry = yield* ToolRegistry.Service
|
||||
|
||||
expect(
|
||||
yield* executeTool(registry, {
|
||||
sessionID,
|
||||
...toolIdentity,
|
||||
call: { type: "tool-call", id: "call-image-fallback", name: "read", input: { path: "pixel.png" } },
|
||||
}),
|
||||
).toMatchObject({
|
||||
type: "content",
|
||||
value: [{ type: "text" }, { type: "media", mediaType: "image/png", data: png }],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects invalid image data returned by the filesystem", () =>
|
||||
Effect.gen(function* () {
|
||||
readResult = new FileSystem.BinaryContent({
|
||||
|
||||
Loading…
Reference in New Issue
Block a user