296 lines
12 KiB
TypeScript
296 lines
12 KiB
TypeScript
import { describe, expect, test } from "bun:test"
|
|
import { Duration, Effect, Fiber, Layer, Schema } from "effect"
|
|
import * as TestClock from "effect/testing/TestClock"
|
|
import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
|
|
import { PermissionV2 } from "@opencode-ai/core/permission"
|
|
import { SessionV2 } from "@opencode-ai/core/session"
|
|
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
|
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
|
import { WebFetchTool } from "@opencode-ai/core/tool/webfetch"
|
|
import { testEffect } from "./lib/effect"
|
|
|
|
const sessionID = SessionV2.ID.make("ses_webfetch_test")
|
|
const requests: Array<{ readonly url: string; readonly headers: Record<string, string> }> = []
|
|
const assertions: PermissionV2.AssertInput[] = []
|
|
const truncations: ToolOutputStore.TruncateInput[] = []
|
|
let respond = (_request: HttpClientRequest.HttpClientRequest) =>
|
|
Effect.succeed(new Response("hello", { headers: { "content-type": "text/plain" } }))
|
|
let truncate = (input: ToolOutputStore.TruncateInput): Effect.Effect<ToolOutputStore.TruncateResult> =>
|
|
Effect.succeed({ content: input.content, truncated: false })
|
|
|
|
const http = Layer.succeed(
|
|
HttpClient.HttpClient,
|
|
HttpClient.make((request) =>
|
|
Effect.sync(() => requests.push({ url: request.url, headers: request.headers })).pipe(
|
|
Effect.andThen(respond(request)),
|
|
Effect.map((response) => HttpClientResponse.fromWeb(request, response)),
|
|
),
|
|
),
|
|
)
|
|
const permission = Layer.succeed(
|
|
PermissionV2.Service,
|
|
PermissionV2.Service.of({
|
|
assert: (input) => Effect.sync(() => assertions.push(input)),
|
|
ask: () => Effect.die("unused"),
|
|
reply: () => Effect.die("unused"),
|
|
get: () => Effect.die("unused"),
|
|
forSession: () => Effect.die("unused"),
|
|
list: () => Effect.die("unused"),
|
|
}),
|
|
)
|
|
const resources = Layer.succeed(
|
|
ToolOutputStore.Service,
|
|
ToolOutputStore.Service.of({
|
|
limits: () => Effect.die("unused"),
|
|
write: () => Effect.die("unused"),
|
|
truncate: (input) => Effect.sync(() => truncations.push(input)).pipe(Effect.andThen(truncate(input))),
|
|
read: () => Effect.die("unused"),
|
|
cleanup: () => Effect.die("unused"),
|
|
}),
|
|
)
|
|
const registry = ToolRegistry.defaultLayer.pipe(Layer.provide(permission))
|
|
const webfetch = WebFetchTool.layer.pipe(Layer.provide(registry), Layer.provide(http), Layer.provide(resources))
|
|
const it = testEffect(Layer.mergeAll(registry, permission, http, resources, webfetch))
|
|
const fetchWebfetch = WebFetchTool.layer.pipe(
|
|
Layer.provide(registry),
|
|
Layer.provide(FetchHttpClient.layer),
|
|
Layer.provide(resources),
|
|
)
|
|
const live = testEffect(Layer.mergeAll(registry, permission, FetchHttpClient.layer, resources, fetchWebfetch))
|
|
|
|
const reset = () => {
|
|
requests.length = 0
|
|
assertions.length = 0
|
|
truncations.length = 0
|
|
respond = () => Effect.succeed(new Response("hello", { headers: { "content-type": "text/plain" } }))
|
|
truncate = (input) => Effect.succeed({ content: input.content, truncated: false })
|
|
}
|
|
|
|
const call = (input: typeof WebFetchTool.Parameters.Type, id = "call-webfetch") => ({
|
|
sessionID,
|
|
call: { type: "tool-call" as const, id, name: "webfetch", input },
|
|
})
|
|
|
|
describe("WebFetchTool helpers", () => {
|
|
test("defaults format and rejects invalid timeout controls", () => {
|
|
const decode = Schema.decodeUnknownSync(WebFetchTool.Parameters)
|
|
expect(decode({ url: "https://example.com" })).toEqual({ url: "https://example.com", format: "markdown" })
|
|
expect(() => decode({ url: "https://example.com", timeout: 0 })).toThrow()
|
|
expect(() => decode({ url: "https://example.com", timeout: WebFetchTool.MAX_TIMEOUT_SECONDS + 1 })).toThrow()
|
|
})
|
|
|
|
test("ports HTML text and markdown conversions without active content", () => {
|
|
const html = "<h1>Hello</h1><script>bad()</script><p>world <strong>wide</strong></p><style>.bad {}</style>"
|
|
expect(WebFetchTool.extractTextFromHTML(html)).toBe("Helloworld wide")
|
|
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("# Hello\n\nworld **wide**")
|
|
})
|
|
})
|
|
|
|
describe("WebFetchTool contribution", () => {
|
|
it.effect("registers and fetches an ordinary hostname HTTP URL without rewriting it", () =>
|
|
Effect.gen(function* () {
|
|
reset()
|
|
const registry = yield* ToolRegistry.Service
|
|
const url = "http://example.com/public"
|
|
|
|
expect((yield* registry.definitions()).map((tool) => tool.name)).toEqual(["webfetch"])
|
|
expect(yield* registry.settle(call({ url, format: "text", timeout: 4 }))).toEqual({
|
|
result: { type: "text", value: "hello" },
|
|
output: {
|
|
structured: { url, contentType: "text/plain", format: "text", output: "hello", truncated: false },
|
|
content: [{ type: "text", text: "hello" }],
|
|
},
|
|
})
|
|
expect(assertions).toEqual([
|
|
{ sessionID, action: "webfetch", resources: [url], save: ["*"], metadata: { url, format: "text", timeout: 4 } },
|
|
])
|
|
expect(requests).toMatchObject([{ url, headers: { accept: expect.stringContaining("text/plain;q=1.0") } }])
|
|
}),
|
|
)
|
|
|
|
it.effect("accepts localhost URLs with the same requested-URL permission check", () =>
|
|
Effect.gen(function* () {
|
|
reset()
|
|
const registry = yield* ToolRegistry.Service
|
|
const url = "http://localhost/private"
|
|
|
|
expect(yield* registry.execute(call({ url, format: "text" }))).toEqual({
|
|
type: "text",
|
|
value: "hello",
|
|
})
|
|
expect(assertions).toEqual([
|
|
{ sessionID, action: "webfetch", resources: [url], save: ["*"], metadata: { url, format: "text" } },
|
|
])
|
|
expect(requests.map((request) => request.url)).toEqual([url])
|
|
}),
|
|
)
|
|
|
|
live.effect("follows redirects while approving only the requested URL", () =>
|
|
Effect.acquireUseRelease(
|
|
Effect.sync(() =>
|
|
Bun.serve({
|
|
port: 0,
|
|
fetch: (request) =>
|
|
new URL(request.url).pathname === "/redirect"
|
|
? new Response("", { status: 302, headers: { location: "/target" } })
|
|
: new Response("redirected", { headers: { "content-type": "text/plain" } }),
|
|
}),
|
|
),
|
|
(server) =>
|
|
Effect.gen(function* () {
|
|
reset()
|
|
const registry = yield* ToolRegistry.Service
|
|
const url = new URL("/redirect", server.url).toString()
|
|
|
|
expect(yield* registry.execute(call({ url, format: "text" }))).toEqual({ type: "text", value: "redirected" })
|
|
expect(assertions).toEqual([
|
|
{ sessionID, action: "webfetch", resources: [url], save: ["*"], metadata: { url, format: "text" } },
|
|
])
|
|
}),
|
|
(server) => Effect.promise(() => server.stop(true)),
|
|
),
|
|
)
|
|
|
|
it.effect("rejects non-HTTP schemes before permission or transport", () =>
|
|
Effect.gen(function* () {
|
|
reset()
|
|
const registry = yield* ToolRegistry.Service
|
|
|
|
expect(yield* registry.execute(call({ url: "file:///etc/passwd", format: "text" }))).toEqual({
|
|
type: "error",
|
|
value: "Unable to fetch file:///etc/passwd",
|
|
})
|
|
expect(assertions).toEqual([])
|
|
expect(requests).toEqual([])
|
|
}),
|
|
)
|
|
|
|
it.effect("converts HTML to requested markdown and text", () =>
|
|
Effect.gen(function* () {
|
|
reset()
|
|
respond = () =>
|
|
Effect.succeed(
|
|
new Response("<h1>Hello</h1><p>world</p><script>bad()</script>", {
|
|
headers: { "content-type": "text/html; charset=utf-8" },
|
|
}),
|
|
)
|
|
const registry = yield* ToolRegistry.Service
|
|
|
|
expect(yield* registry.execute(call({ url: "https://1.1.1.1", format: "markdown" }))).toEqual({
|
|
type: "text",
|
|
value: "# Hello\n\nworld",
|
|
})
|
|
expect(yield* registry.execute(call({ url: "https://1.1.1.1", format: "text" }))).toEqual({
|
|
type: "text",
|
|
value: "Helloworld",
|
|
})
|
|
}),
|
|
)
|
|
|
|
it.effect("exposes managed overflow through an opaque resource URI", () =>
|
|
Effect.gen(function* () {
|
|
reset()
|
|
truncate = (input) =>
|
|
Effect.succeed({
|
|
content: "HEAD\n\n... output truncated; full content available as tool-output://opaque ...\n\nTAIL",
|
|
truncated: true,
|
|
resource: new ToolOutputStore.Resource({
|
|
uri: "tool-output://opaque",
|
|
mime: input.mime ?? "text/plain",
|
|
size: input.content.length,
|
|
}),
|
|
})
|
|
const registry = yield* ToolRegistry.Service
|
|
const settled = yield* registry.settle(call({ url: "https://1.1.1.1", format: "html" }, "call-overflow"))
|
|
|
|
expect(settled.result).toMatchObject({ type: "text", value: expect.stringContaining("tool-output://opaque") })
|
|
expect(settled.output?.structured).toMatchObject({
|
|
truncated: true,
|
|
resource: { uri: "tool-output://opaque", mime: "text/html" },
|
|
})
|
|
expect(truncations).toEqual([{ sessionID, toolCallID: "call-overflow", content: "hello", mime: "text/html" }])
|
|
}),
|
|
)
|
|
|
|
it.effect("rejects declared and streamed oversized bodies", () =>
|
|
Effect.gen(function* () {
|
|
reset()
|
|
const registry = yield* ToolRegistry.Service
|
|
respond = () =>
|
|
Effect.succeed(
|
|
new Response("small", {
|
|
headers: { "content-type": "text/plain", "content-length": String(WebFetchTool.MAX_RESPONSE_BYTES + 1) },
|
|
}),
|
|
)
|
|
expect(yield* registry.execute(call({ url: "https://1.1.1.1/declared", format: "text" }))).toEqual({
|
|
type: "error",
|
|
value: "Unable to fetch https://1.1.1.1/declared",
|
|
})
|
|
|
|
respond = () =>
|
|
Effect.succeed(
|
|
new Response("x".repeat(WebFetchTool.MAX_RESPONSE_BYTES + 1), { headers: { "content-type": "text/plain" } }),
|
|
)
|
|
expect(yield* registry.execute(call({ url: "https://1.1.1.1/streamed", format: "text" }))).toEqual({
|
|
type: "error",
|
|
value: "Unable to fetch https://1.1.1.1/streamed",
|
|
})
|
|
}),
|
|
)
|
|
|
|
it.effect("keeps images and files unsupported until typed settlement can carry attachments", () =>
|
|
Effect.gen(function* () {
|
|
reset()
|
|
const registry = yield* ToolRegistry.Service
|
|
respond = () => Effect.succeed(new Response("png", { headers: { "content-type": "image/png" } }))
|
|
expect(yield* registry.execute(call({ url: "https://1.1.1.1/image", format: "html" }))).toEqual({
|
|
type: "error",
|
|
value: "Unable to fetch https://1.1.1.1/image",
|
|
})
|
|
|
|
respond = () => Effect.succeed(new Response("pdf", { headers: { "content-type": "application/pdf" } }))
|
|
expect(yield* registry.execute(call({ url: "https://1.1.1.1/file", format: "html" }))).toEqual({
|
|
type: "error",
|
|
value: "Unable to fetch https://1.1.1.1/file",
|
|
})
|
|
expect(truncations).toEqual([])
|
|
}),
|
|
)
|
|
|
|
it.effect("retries Cloudflare challenges with an honest user agent", () =>
|
|
Effect.gen(function* () {
|
|
reset()
|
|
let count = 0
|
|
respond = () =>
|
|
Effect.succeed(
|
|
++count === 1
|
|
? new Response("challenge", { status: 403, headers: { "cf-mitigated": "challenge" } })
|
|
: new Response("ok", { headers: { "content-type": "text/plain" } }),
|
|
)
|
|
const registry = yield* ToolRegistry.Service
|
|
|
|
expect(yield* registry.execute(call({ url: "https://1.1.1.1", format: "text" }))).toEqual({
|
|
type: "text",
|
|
value: "ok",
|
|
})
|
|
expect(requests).toHaveLength(2)
|
|
expect(requests[0]?.headers["user-agent"]).toContain("Mozilla/5.0")
|
|
expect(requests[1]?.headers["user-agent"]).toBe("opencode")
|
|
}),
|
|
)
|
|
|
|
it.effect("times out stalled requests", () =>
|
|
Effect.gen(function* () {
|
|
reset()
|
|
respond = () => Effect.never
|
|
const registry = yield* ToolRegistry.Service
|
|
const fiber = yield* registry
|
|
.execute(call({ url: "https://1.1.1.1/slow", format: "text", timeout: 1 }))
|
|
.pipe(Effect.forkChild)
|
|
yield* TestClock.adjust(Duration.seconds(1))
|
|
|
|
expect(yield* Fiber.join(fiber)).toEqual({ type: "error", value: "Unable to fetch https://1.1.1.1/slow" })
|
|
}),
|
|
)
|
|
})
|