From 10d1e04e9b83116c5f3ab2dc8ab5562bc16a2575 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Sat, 6 Jun 2026 21:24:54 -0400 Subject: [PATCH] fix(core): isolate image normalization (#31165) --- packages/core/src/image.ts | 72 ++++++++++++++ packages/core/src/image/photon.ts | 94 ++++++++++++++++++ packages/core/src/location-layer.ts | 18 +++- packages/core/src/tool/read.ts | 138 ++------------------------- packages/core/test/tool-read.test.ts | 43 ++++++++- 5 files changed, 230 insertions(+), 135 deletions(-) create mode 100644 packages/core/src/image.ts create mode 100644 packages/core/src/image/photon.ts diff --git a/packages/core/src/image.ts b/packages/core/src/image.ts new file mode 100644 index 000000000..52661e89b --- /dev/null +++ b/packages/core/src/image.ts @@ -0,0 +1,72 @@ +export * as Image from "./image" + +import { Context, Effect, Layer, Schema } from "effect" +import { Config } from "./config" +import { FileSystem } from "./filesystem" + +export class ResizerUnavailableError extends Schema.TaggedErrorClass()( + "Image.ResizerUnavailableError", + {}, +) {} + +export class DecodeError extends Schema.TaggedErrorClass()("Image.DecodeError", { + resource: Schema.String, +}) { + override get message() { + return `Image could not be decoded: ${this.resource}` + } +} + +export class SizeError extends Schema.TaggedErrorClass()("Image.SizeError", { + resource: Schema.String, + width: Schema.Number, + height: Schema.Number, + bytes: Schema.Number, + maxWidth: Schema.Number, + maxHeight: Schema.Number, + maxBytes: Schema.Number, +}) { + override get message() { + return `Image ${this.resource} is ${this.width}x${this.height} with base64 size ${this.bytes}, exceeding configured limits ${this.maxWidth}x${this.maxHeight}/${this.maxBytes} bytes` + } +} + +export interface Interface { + readonly normalize: ( + resource: string, + content: FileSystem.BinaryContent, + ) => Effect.Effect +} + +export class Service extends Context.Service()("@opencode/Image") {} + +export const layer = Layer.effect( + Service, + Effect.gen(function* () { + const config = yield* Config.Service + const loadAdapter = yield* Effect.cached( + Effect.tryPromise({ + try: () => import("./image/photon"), + catch: () => new ResizerUnavailableError(), + }).pipe(Effect.flatMap((adapter) => adapter.make)), + ) + const normalize = Effect.fn("Image.normalize")(function* (resource: string, content: FileSystem.BinaryContent) { + const image = Object.assign( + {}, + ...(yield* config.entries()).flatMap((entry) => + entry.type === "document" && entry.info.attachments?.image ? [entry.info.attachments.image] : [], + ), + ) + const normalize = yield* loadAdapter + return yield* normalize(resource, content, { + autoResize: image.auto_resize ?? true, + maxWidth: image.max_width ?? 2_000, + maxHeight: image.max_height ?? 2_000, + maxBase64Bytes: image.max_base64_bytes ?? 5 * 1024 * 1024, + }) + }) + return Service.of({ normalize }) + }), +) + +export const locationLayer = layer.pipe(Layer.provide(Config.locationLayer)) diff --git a/packages/core/src/image/photon.ts b/packages/core/src/image/photon.ts new file mode 100644 index 000000000..b78ea7930 --- /dev/null +++ b/packages/core/src/image/photon.ts @@ -0,0 +1,94 @@ +// @ts-ignore Bun's static file import is embedded by `bun build --compile`; some consumers also declare *.wasm. +import photonWasm from "@silvia-odwyer/photon-node/photon_rs_bg.wasm" with { type: "file" } +import { Effect } from "effect" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { FileSystem } from "../filesystem" +import { DecodeError, ResizerUnavailableError, SizeError } from "../image" + +const JPEG_QUALITIES = [80, 85, 70, 55, 40] + +export const make = Effect.gen(function* () { + ;(globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }).__OPENCODE_PHOTON_WASM_PATH = + path.isAbsolute(photonWasm) ? photonWasm : fileURLToPath(new URL(photonWasm, import.meta.url)) + const loadPhoton = yield* Effect.cached( + Effect.tryPromise({ + try: () => import("@silvia-odwyer/photon-node"), + catch: () => new ResizerUnavailableError(), + }), + ) + return Effect.fn("Image.Photon.normalize")(function* ( + resource: string, + content: FileSystem.BinaryContent, + limits: { + readonly autoResize: boolean + readonly maxWidth: number + readonly maxHeight: number + readonly maxBase64Bytes: number + }, + ) { + const photon = yield* loadPhoton + const decoded = yield* Effect.try({ + try: () => photon.PhotonImage.new_from_byteslice(Buffer.from(content.content, "base64")), + catch: () => new DecodeError({ resource }), + }) + try { + const width = decoded.get_width() + const height = decoded.get_height() + const bytes = Buffer.byteLength(content.content, "utf-8") + if (width <= limits.maxWidth && height <= limits.maxHeight && bytes <= limits.maxBase64Bytes) return content + if (!limits.autoResize) + return yield* new SizeError({ + resource, + width, + height, + bytes, + maxWidth: limits.maxWidth, + maxHeight: limits.maxHeight, + maxBytes: limits.maxBase64Bytes, + }) + const scale = Math.min(1, limits.maxWidth / width, limits.maxHeight / height) + const sizes = Array.from({ length: 32 }).reduce>((acc) => { + const previous = acc.at(-1) ?? { + width: Math.max(1, Math.round(width * scale)), + height: Math.max(1, Math.round(height * scale)), + } + const next = + acc.length === 0 + ? previous + : { + width: previous.width === 1 ? 1 : Math.max(1, Math.floor(previous.width * 0.75)), + height: previous.height === 1 ? 1 : Math.max(1, Math.floor(previous.height * 0.75)), + } + return acc.some((item) => item.width === next.width && item.height === next.height) ? acc : [...acc, next] + }, []) + for (const size of sizes) { + const resized = photon.resize(decoded, size.width, size.height, photon.SamplingFilter.Lanczos3) + try { + const encoders: Array Uint8Array]> = [ + ["image/png", () => resized.get_bytes()], + ...JPEG_QUALITIES.map((quality) => ["image/jpeg", () => resized.get_bytes_jpeg(quality)] as const), + ] + for (const [mime, encode] of encoders) { + const candidate = Buffer.from(encode()).toString("base64") + if (Buffer.byteLength(candidate, "utf-8") <= limits.maxBase64Bytes) + return new FileSystem.BinaryContent({ type: "binary", content: candidate, encoding: "base64", mime }) + } + } finally { + resized.free() + } + } + return yield* new SizeError({ + resource, + width, + height, + bytes, + maxWidth: limits.maxWidth, + maxHeight: limits.maxHeight, + maxBytes: limits.maxBase64Bytes, + }) + } finally { + decoded.free() + } + }) +}) diff --git a/packages/core/src/location-layer.ts b/packages/core/src/location-layer.ts index 471be0903..e53cad579 100644 --- a/packages/core/src/location-layer.ts +++ b/packages/core/src/location-layer.ts @@ -28,6 +28,7 @@ import { Pty } from "./pty" import { SkillV2 } from "./skill" import { SkillGuidance } from "./skill/guidance" import { BuiltInTools } from "./tool/builtins" +import { Image } from "./image" import { ToolRegistry } from "./tool/registry" import { ApplicationTools } from "./tool/application-tools" import { ToolOutputStore } from "./tool-output-store" @@ -71,6 +72,7 @@ export class LocationServiceMap extends LayerMap.Service()(" Layer.provide(base), ) const services = Layer.mergeAll(base, resources, permissionsAndTools) + const image = Image.layer.pipe(Layer.provide(services)) const mutation = FileMutation.locationLayer.pipe(Layer.provide(services)) const searches = LocationSearch.layer.pipe(Layer.provide(Ripgrep.layer), Layer.provide(services)) const skillGuidance = SkillGuidance.locationLayer.pipe(Layer.provide(services)) @@ -83,6 +85,7 @@ export class LocationServiceMap extends LayerMap.Service()(" Layer.provide(resources), Layer.provide(todos), Layer.provide(questions), + Layer.provide(image), ) const model = SessionRunnerModel.locationLayer.pipe(Layer.provide(services)) const runner = SessionRunnerLLM.defaultLayer.pipe( @@ -90,9 +93,18 @@ export class LocationServiceMap extends LayerMap.Service()(" Layer.provide(model), Layer.provide(skillGuidance), ) - return Layer.mergeAll(services, mutation, searches, resources, todos, questions, model, runner, builtInTools).pipe( - Layer.fresh, - ) + return Layer.mergeAll( + services, + image, + mutation, + searches, + resources, + todos, + questions, + model, + runner, + builtInTools, + ).pipe(Layer.fresh) }, idleTimeToLive: "60 minutes", dependencies: [ diff --git a/packages/core/src/tool/read.ts b/packages/core/src/tool/read.ts index 2ebb376b8..b3a610398 100644 --- a/packages/core/src/tool/read.ts +++ b/packages/core/src/tool/read.ts @@ -1,47 +1,15 @@ export * as ReadTool from "./read" import { ToolFailure } from "@opencode-ai/llm" -// @ts-ignore Bun's static file import is embedded by `bun build --compile`; some consumers also declare *.wasm. -import photonWasm from "@silvia-odwyer/photon-node/photon_rs_bg.wasm" with { type: "file" } import { Effect, Layer, Schema } from "effect" -import path from "node:path" -import { fileURLToPath } from "node:url" -import { Config } from "../config" import { FileSystem } from "../filesystem" +import { Image } from "../image" import { PermissionV2 } from "../permission" import { Tool } from "./tool" import { Tools } from "./tools" export const name = "read" const SUPPORTED_IMAGE_MIMES = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"]) -const MAX_IMAGE_BASE64_BYTES = 5 * 1024 * 1024 -const MAX_IMAGE_WIDTH = 2_000 -const MAX_IMAGE_HEIGHT = 2_000 -const JPEG_QUALITIES = [80, 85, 70, 55, 40] - -class ImageDecodeError extends Error { - constructor(readonly resource: string) { - super(`Image could not be decoded: ${resource}`) - this.name = "ImageDecodeError" - } -} - -class ImageSizeError extends Error { - constructor( - readonly resource: string, - readonly width: number, - readonly height: number, - readonly bytes: number, - readonly maxWidth: number, - readonly maxHeight: number, - readonly maxBytes: number, - ) { - super( - `Image ${resource} is ${width}x${height} with base64 size ${bytes}, exceeding configured limits ${maxWidth}x${maxHeight}/${maxBytes} bytes`, - ) - this.name = "ImageSizeError" - } -} const LocationInput = Schema.Struct({ ...FileSystem.ReadInput.fields, offset: FileSystem.ListPageInput.fields.offset.annotate({ @@ -58,14 +26,8 @@ export const layer = Layer.effectDiscard( Effect.gen(function* () { const tools = yield* Tools.Service const filesystem = yield* FileSystem.Service - const config = yield* Config.Service + const image = yield* Image.Service const permission = yield* PermissionV2.Service - const loadPhoton = yield* Effect.cached( - Effect.sync(() => { - ;(globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }).__OPENCODE_PHOTON_WASM_PATH = - path.isAbsolute(photonWasm) ? photonWasm : fileURLToPath(new URL(photonWasm, import.meta.url)) - }).pipe(Effect.andThen(() => Effect.promise(() => import("@silvia-odwyer/photon-node")))), - ) yield* tools .register({ @@ -98,95 +60,9 @@ export const layer = Layer.effectDiscard( limit: input.limit, }) if (content.type === "binary" && SUPPORTED_IMAGE_MIMES.has(content.mime)) { - const mime = content.mime - const base64 = content.content - const image = Object.assign( - {}, - ...(yield* config.entries()).flatMap((entry) => - entry.type === "document" && entry.info.attachments?.image ? [entry.info.attachments.image] : [], - ), - ) - const limits = { - autoResize: image.auto_resize ?? true, - maxWidth: image.max_width ?? MAX_IMAGE_WIDTH, - maxHeight: image.max_height ?? MAX_IMAGE_HEIGHT, - maxBase64Bytes: image.max_base64_bytes ?? MAX_IMAGE_BASE64_BYTES, - } - const photon = yield* loadPhoton - const decoded = yield* Effect.try({ - try: () => photon.PhotonImage.new_from_byteslice(Buffer.from(base64, "base64")), - catch: () => new ImageDecodeError(resolved.resource), - }) - try { - const width = decoded.get_width() - const height = decoded.get_height() - const bytes = Buffer.byteLength(base64, "utf-8") - if (width <= limits.maxWidth && height <= limits.maxHeight && bytes <= limits.maxBase64Bytes) - return new FileSystem.BinaryContent({ type: "binary", content: base64, encoding: "base64", mime }) - if (!limits.autoResize) - return yield* Effect.fail( - new ImageSizeError( - resolved.resource, - width, - height, - bytes, - limits.maxWidth, - limits.maxHeight, - limits.maxBase64Bytes, - ), - ) - const scale = Math.min(1, limits.maxWidth / width, limits.maxHeight / height) - const sizes = Array.from({ length: 32 }).reduce>((acc) => { - const previous = acc.at(-1) ?? { - width: Math.max(1, Math.round(width * scale)), - height: Math.max(1, Math.round(height * scale)), - } - const next = - acc.length === 0 - ? previous - : { - width: previous.width === 1 ? 1 : Math.max(1, Math.floor(previous.width * 0.75)), - height: previous.height === 1 ? 1 : Math.max(1, Math.floor(previous.height * 0.75)), - } - return acc.some((item) => item.width === next.width && item.height === next.height) - ? acc - : [...acc, next] - }, []) - for (const size of sizes) { - const resized = photon.resize(decoded, size.width, size.height, photon.SamplingFilter.Lanczos3) - try { - const candidate = [ - { content: Buffer.from(resized.get_bytes()).toString("base64"), mime: "image/png" }, - ...JPEG_QUALITIES.map((quality) => ({ - content: Buffer.from(resized.get_bytes_jpeg(quality)).toString("base64"), - mime: "image/jpeg", - })), - ].find((item) => Buffer.byteLength(item.content, "utf-8") <= limits.maxBase64Bytes) - if (candidate) - return new FileSystem.BinaryContent({ - type: "binary", - content: candidate.content, - encoding: "base64", - mime: candidate.mime, - }) - } finally { - resized.free() - } - } - return yield* Effect.fail( - new ImageSizeError( - resolved.resource, - width, - height, - bytes, - limits.maxWidth, - limits.maxHeight, - limits.maxBase64Bytes, - ), - ) - } finally { - decoded.free() - } + return yield* image + .normalize(resolved.resource, content) + .pipe(Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(content))) } if (content.type === "binary") return yield* Effect.fail(new FileSystem.BinaryFileError(resolved.resource)) @@ -196,8 +72,8 @@ export const layer = Layer.effectDiscard( const message = error instanceof FileSystem.BinaryFileError || error instanceof FileSystem.MediaIngestLimitError || - error instanceof ImageDecodeError || - error instanceof ImageSizeError + error instanceof Image.DecodeError || + error instanceof Image.SizeError ? error.message : `Unable to read ${input.path}` return new ToolFailure({ message }) diff --git a/packages/core/test/tool-read.test.ts b/packages/core/test/tool-read.test.ts index ff22dd578..4b9ad369a 100644 --- a/packages/core/test/tool-read.test.ts +++ b/packages/core/test/tool-read.test.ts @@ -3,6 +3,7 @@ import { Effect, Exit, Layer } from "effect" import { Config } from "@opencode-ai/core/config" import { ConfigAttachments } from "@opencode-ai/core/config/attachments" import { FileSystem } from "@opencode-ai/core/filesystem" +import { Image } from "@opencode-ai/core/image" import { PermissionV2 } from "@opencode-ai/core/permission" import { SessionV2 } from "@opencode-ai/core/session" import { ToolRegistry } from "@opencode-ai/core/tool/registry" @@ -75,13 +76,29 @@ const permission = Layer.succeed( ) const registry = ToolRegistry.defaultLayer.pipe(Layer.provide(permission)) const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () => Effect.succeed(configEntries) })) +const image = Image.layer.pipe(Layer.provide(config)) +const unavailableImage = Layer.succeed( + Image.Service, + Image.Service.of({ normalize: () => Effect.fail(new Image.ResizerUnavailableError()) }), +) const read = ReadTool.layer.pipe( Layer.provide(registry), Layer.provide(filesystem), Layer.provide(permission), Layer.provide(config), + Layer.provide(image), +) +const it = testEffect(Layer.mergeAll(registry, filesystem, permission, config, image, read)) +const unavailableRead = ReadTool.layer.pipe( + Layer.provide(registry), + Layer.provide(filesystem), + Layer.provide(permission), + Layer.provide(config), + Layer.provide(unavailableImage), +) +const itWithoutResizer = testEffect( + Layer.mergeAll(registry, filesystem, permission, config, unavailableImage, unavailableRead), ) -const it = testEffect(Layer.mergeAll(registry, filesystem, permission, config, read)) const sessionID = SessionV2.ID.make("ses_read_tool_test") describe("ReadTool", () => { @@ -188,6 +205,30 @@ describe("ReadTool", () => { }), ) + itWithoutResizer.effect("returns the original image when the resizer is unavailable", () => + Effect.gen(function* () { + const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" + readResult = new FileSystem.BinaryContent({ + type: "binary", + content: png, + encoding: "base64", + mime: "image/png", + }) + const registry = yield* ToolRegistry.Service + + expect( + yield* executeTool(registry, { + sessionID, + ...toolIdentity, + call: { type: "tool-call", id: "call-image-fallback", name: "read", input: { path: "pixel.png" } }, + }), + ).toMatchObject({ + type: "content", + value: [{ type: "text" }, { type: "media", mediaType: "image/png", data: png }], + }) + }), + ) + it.effect("rejects invalid image data returned by the filesystem", () => Effect.gen(function* () { readResult = new FileSystem.BinaryContent({